Benchmark #215
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| primary_driver: | |
| description: "Primary driver" | |
| required: true | |
| type: choice | |
| options: | |
| - spring-data-valkey | |
| - spring-data-redis | |
| - valkey-glide | |
| - jedis | |
| - lettuce | |
| - redisson | |
| default: "spring-data-valkey" | |
| secondary_driver: | |
| description: "Secondary driver (for spring-data-* only, ignored for others)" | |
| required: false | |
| type: choice | |
| options: | |
| - valkey-glide | |
| - jedis | |
| - lettuce | |
| - none | |
| default: "valkey-glide" | |
| workload: | |
| description: "Workload config" | |
| required: true | |
| type: choice | |
| options: | |
| - reference-workload-10-client | |
| - reference-workload-1-client | |
| default: "reference-workload-10-client" | |
| primary_version: | |
| description: "Primary driver version (version string or commit ID). Leave empty to use default." | |
| required: false | |
| type: string | |
| default: "" | |
| secondary_version: | |
| description: "Secondary driver version (e.g., valkey-glide commit). Leave empty to use default." | |
| required: false | |
| type: string | |
| default: "" | |
| job_id_prefix: | |
| description: 'Optional job ID prefix (e.g., "nightly", "pr-123")' | |
| required: false | |
| type: string | |
| default: "" | |
| custom_driver_config: | |
| description: "Custom driver config JSON (overrides driver selection)" | |
| required: false | |
| type: string | |
| default: "" | |
| custom_workload_config: | |
| description: "Custom workload config JSON (overrides workload selection)" | |
| required: false | |
| type: string | |
| default: "" | |
| network_delay: | |
| description: "Network delay with unit, e.g. '1ms' or '500us' (tc netem). Leave empty to disable." | |
| required: false | |
| type: string | |
| default: "" | |
| network_jitter: | |
| description: "Network jitter with unit, e.g. '1ms' or '100us' (tc netem). Requires network_delay." | |
| required: false | |
| type: string | |
| default: "" | |
| network_delay_distribution: | |
| description: "Jitter distribution (tc netem). Requires jitter. Options: normal, pareto, paretonormal." | |
| required: false | |
| type: choice | |
| options: | |
| - "" | |
| - normal | |
| - pareto | |
| - paretonormal | |
| default: "" | |
| env: | |
| RESP_BENCH_REPO: "https://github.com/ikolomi/resp-bench.git" | |
| jobs: | |
| benchmark: | |
| runs-on: [self-hosted, Linux, x86, ephemeral, metal] | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Validate EC2 type and CPU model | |
| run: | | |
| # c5.metal: Intel Xeon Platinum 8275CL (Cascade Lake), 96 vCPUs, 2 NUMA nodes | |
| EXPECTED_INSTANCE_TYPE="c5.metal" | |
| EXPECTED_CPU="8275CL" | |
| EXPECTED_STEPPING="7" | |
| EXPECTED_VCPUS="96" | |
| TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds:60") | |
| INSTANCE_TYPE=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-type) | |
| CPU_MODEL=$(grep 'model name' /proc/cpuinfo | head -1 | sed 's/.*: //') | |
| CPU_STEPPING=$(lscpu | grep '^Stepping:' | awk '{print $2}') | |
| CPU_COUNT=$(nproc) | |
| echo "Instance type: $INSTANCE_TYPE" | |
| echo "CPU: $CPU_MODEL" | |
| echo "Stepping: $CPU_STEPPING" | |
| echo "vCPUs: $CPU_COUNT" | |
| lscpu | grep -E 'Model name|Stepping|CPU max MHz|CPU\(s\):|NUMA' | |
| ERRORS="" | |
| if [ "$INSTANCE_TYPE" != "$EXPECTED_INSTANCE_TYPE" ]; then | |
| ERRORS="${ERRORS}Instance type: expected '$EXPECTED_INSTANCE_TYPE', got '$INSTANCE_TYPE'\n" | |
| fi | |
| if ! echo "$CPU_MODEL" | grep -q "$EXPECTED_CPU"; then | |
| ERRORS="${ERRORS}CPU model: expected '$EXPECTED_CPU', got '$CPU_MODEL'\n" | |
| fi | |
| if [ "$CPU_STEPPING" != "$EXPECTED_STEPPING" ]; then | |
| ERRORS="${ERRORS}Stepping: expected '$EXPECTED_STEPPING', got '$CPU_STEPPING'\n" | |
| fi | |
| if [ "$CPU_COUNT" != "$EXPECTED_VCPUS" ]; then | |
| ERRORS="${ERRORS}vCPU count: expected '$EXPECTED_VCPUS', got '$CPU_COUNT'\n" | |
| fi | |
| if [ -n "$ERRORS" ]; then | |
| echo "ERROR: Hardware validation failed:" | |
| echo -e "$ERRORS" | |
| echo "This machine does not match expected c5.metal specs. Failing early to avoid inconsistent benchmark results." | |
| exit 1 | |
| fi | |
| echo "✓ Hardware validated: $CPU_MODEL, stepping $CPU_STEPPING, $CPU_COUNT vCPUs" | |
| - uses: actions/checkout@v4 | |
| - name: Set up Java 21 and Maven | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y openjdk-21-jdk-headless maven python3-pip \ | |
| sysstat numactl \ | |
| gcc pkg-config openssl libssl-dev unzip cmake | |
| sudo apt-get install -y linux-tools-$(uname -r) 2>/dev/null || \ | |
| sudo apt-get install -y linux-tools-generic 2>/dev/null || true | |
| java -version | |
| mvn -version | |
| - name: Clone resp-bench | |
| id: resp-bench | |
| run: | | |
| echo "Cloning resp-bench repository..." | |
| git clone --depth 1 ${{ env.RESP_BENCH_REPO }} resp-bench | |
| RESP_BENCH_COMMIT=$(git -C resp-bench rev-parse HEAD) | |
| echo "commit=$RESP_BENCH_COMMIT" >> $GITHUB_OUTPUT | |
| VALKEY_SERVER_VERSION=$(grep '^SERVER_VERSION?=' resp-bench/Makefile | cut -d= -f2) | |
| echo "valkey_version=$VALKEY_SERVER_VERSION" >> $GITHUB_OUTPUT | |
| echo "✓ resp-bench cloned (commit: ${RESP_BENCH_COMMIT:0:7}, valkey: $VALKEY_SERVER_VERSION)" | |
| - name: Resolve inputs | |
| id: inputs | |
| run: | | |
| PRIMARY_DRIVER="${{ github.event.inputs.primary_driver }}" | |
| SECONDARY_DRIVER="${{ github.event.inputs.secondary_driver }}" | |
| WORKLOAD="${{ github.event.inputs.workload }}" | |
| PRIMARY_VERSION="${{ github.event.inputs.primary_version }}" | |
| SECONDARY_VERSION="${{ github.event.inputs.secondary_version }}" | |
| JOB_ID_PREFIX="${{ github.event.inputs.job_id_prefix }}" | |
| NETWORK_DELAY="${{ github.event.inputs.network_delay }}" | |
| NETWORK_JITTER="${{ github.event.inputs.network_jitter }}" | |
| NETWORK_DELAY_DISTRIBUTION="${{ github.event.inputs.network_delay_distribution }}" | |
| # Validate tc netem input combinations | |
| if [ -n "$NETWORK_DELAY_DISTRIBUTION" ] && [ -z "$NETWORK_JITTER" ]; then | |
| echo "ERROR: network_delay_distribution requires network_jitter" | |
| exit 1 | |
| fi | |
| if [ -n "$NETWORK_JITTER" ] && [ -z "$NETWORK_DELAY" ]; then | |
| echo "ERROR: network_jitter requires network_delay" | |
| exit 1 | |
| fi | |
| # Validate unit format (number followed by ms or us) | |
| for val in "$NETWORK_DELAY" "$NETWORK_JITTER"; do | |
| if [ -n "$val" ] && ! [[ "$val" =~ ^[0-9]+(ms|us)$ ]]; then | |
| echo "ERROR: invalid tc netem value '$val' — expected format: <number>ms or <number>us" | |
| exit 1 | |
| fi | |
| done | |
| echo "primary_driver=$PRIMARY_DRIVER" >> $GITHUB_OUTPUT | |
| echo "secondary_driver=$SECONDARY_DRIVER" >> $GITHUB_OUTPUT | |
| echo "workload=$WORKLOAD" >> $GITHUB_OUTPUT | |
| echo "primary_version=$PRIMARY_VERSION" >> $GITHUB_OUTPUT | |
| echo "secondary_version=$SECONDARY_VERSION" >> $GITHUB_OUTPUT | |
| echo "job_id_prefix=$JOB_ID_PREFIX" >> $GITHUB_OUTPUT | |
| echo "network_delay=$NETWORK_DELAY" >> $GITHUB_OUTPUT | |
| echo "network_jitter=$NETWORK_JITTER" >> $GITHUB_OUTPUT | |
| echo "network_delay_distribution=$NETWORK_DELAY_DISTRIBUTION" >> $GITHUB_OUTPUT | |
| echo "========================================" | |
| echo "Primary driver: $PRIMARY_DRIVER" | |
| echo "Secondary driver: $SECONDARY_DRIVER" | |
| echo "Workload: $WORKLOAD" | |
| echo "Primary version: '${PRIMARY_VERSION}'" | |
| echo "Secondary version: '${SECONDARY_VERSION}'" | |
| echo "Job ID prefix: '${JOB_ID_PREFIX}'" | |
| echo "Network delay: '${NETWORK_DELAY}' jitter='${NETWORK_JITTER}' distribution='${NETWORK_DELAY_DISTRIBUTION}'" | |
| echo "========================================" | |
| - name: Validate configs exist | |
| id: validate | |
| run: | | |
| PRIMARY_DRIVER="${{ steps.inputs.outputs.primary_driver }}" | |
| SECONDARY_DRIVER="${{ steps.inputs.outputs.secondary_driver }}" | |
| WORKLOAD="${{ steps.inputs.outputs.workload }}" | |
| CONFIG_DIR=".github/workflows/benchmark_configs" | |
| CUSTOM_CONFIG_DIR="/tmp/custom_configs" | |
| mkdir -p "$CUSTOM_CONFIG_DIR" | |
| # Handle custom driver config | |
| CUSTOM_DRIVER_CONFIG_INPUT='${{ github.event.inputs.custom_driver_config }}' | |
| if [ -n "$CUSTOM_DRIVER_CONFIG_INPUT" ]; then | |
| echo "Using custom driver config..." | |
| DRIVER_CONFIG="$CUSTOM_CONFIG_DIR/custom-driver.json" | |
| printf '%s\n' "$CUSTOM_DRIVER_CONFIG_INPUT" > "$DRIVER_CONFIG" | |
| # Validate JSON | |
| if ! jq . "$DRIVER_CONFIG" > /dev/null 2>&1; then | |
| echo "ERROR: Custom driver config is not valid JSON" | |
| echo "Content written:" | |
| cat "$DRIVER_CONFIG" | |
| exit 1 | |
| fi | |
| DRIVER="custom-driver" | |
| echo "✓ Custom driver config validated" | |
| else | |
| # Resolve driver config filename based on primary and secondary driver | |
| if [[ "$PRIMARY_DRIVER" == spring-data-* ]]; then | |
| # spring-data-* drivers require a secondary driver | |
| if [ "$SECONDARY_DRIVER" = "none" ] || [ -z "$SECONDARY_DRIVER" ]; then | |
| echo "ERROR: $PRIMARY_DRIVER requires a secondary driver (valkey-glide, jedis, or lettuce)" | |
| exit 1 | |
| fi | |
| # spring-data-redis cannot use valkey-glide | |
| if [ "$PRIMARY_DRIVER" = "spring-data-redis" ] && [ "$SECONDARY_DRIVER" = "valkey-glide" ]; then | |
| echo "ERROR: spring-data-redis does not support valkey-glide. Use jedis or lettuce instead." | |
| exit 1 | |
| fi | |
| # Map secondary driver name to filename convention (valkey-glide -> glide) | |
| SECONDARY_IN_FILENAME="$SECONDARY_DRIVER" | |
| if [ "$SECONDARY_DRIVER" = "valkey-glide" ]; then | |
| SECONDARY_IN_FILENAME="glide" | |
| fi | |
| DRIVER="${PRIMARY_DRIVER}-${SECONDARY_IN_FILENAME}-standalone" | |
| else | |
| # Standalone drivers (valkey-glide, jedis, lettuce, redisson) | |
| DRIVER="${PRIMARY_DRIVER}-standalone" | |
| if [ "$SECONDARY_DRIVER" != "none" ] && [ -n "$SECONDARY_DRIVER" ]; then | |
| echo "Note: secondary_driver '$SECONDARY_DRIVER' ignored for standalone driver '$PRIMARY_DRIVER'" | |
| fi | |
| fi | |
| DRIVER_CONFIG="${CONFIG_DIR}/drivers/${DRIVER}.json" | |
| if [ ! -f "$DRIVER_CONFIG" ]; then | |
| echo "ERROR: Driver config not found: $DRIVER_CONFIG" | |
| echo "Available driver configs:" | |
| ls -la "${CONFIG_DIR}/drivers/" | |
| exit 1 | |
| fi | |
| echo "✓ Driver config: $DRIVER_CONFIG" | |
| fi | |
| # Handle custom workload config | |
| CUSTOM_WORKLOAD_CONFIG_INPUT='${{ github.event.inputs.custom_workload_config }}' | |
| if [ -n "$CUSTOM_WORKLOAD_CONFIG_INPUT" ]; then | |
| echo "Using custom workload config..." | |
| WORKLOAD_CONFIG="$CUSTOM_CONFIG_DIR/custom-workload.json" | |
| printf '%s\n' "$CUSTOM_WORKLOAD_CONFIG_INPUT" > "$WORKLOAD_CONFIG" | |
| # Validate JSON | |
| if ! jq . "$WORKLOAD_CONFIG" > /dev/null 2>&1; then | |
| echo "ERROR: Custom workload config is not valid JSON" | |
| echo "Content written:" | |
| cat "$WORKLOAD_CONFIG" | |
| exit 1 | |
| fi | |
| WORKLOAD="custom-workload" | |
| echo "✓ Custom workload config validated" | |
| else | |
| WORKLOAD_CONFIG="${CONFIG_DIR}/workloads/${WORKLOAD}.json" | |
| if [ ! -f "$WORKLOAD_CONFIG" ]; then | |
| echo "ERROR: Workload config not found: $WORKLOAD_CONFIG" | |
| echo "Available workload configs:" | |
| ls -la "${CONFIG_DIR}/workloads/" | |
| exit 1 | |
| fi | |
| echo "✓ Workload config: $WORKLOAD_CONFIG" | |
| fi | |
| echo "driver=$DRIVER" >> $GITHUB_OUTPUT | |
| echo "driver_config=$DRIVER_CONFIG" >> $GITHUB_OUTPUT | |
| echo "workload=$WORKLOAD" >> $GITHUB_OUTPUT | |
| echo "workload_config=$WORKLOAD_CONFIG" >> $GITHUB_OUTPUT | |
| echo "Driver config contents:" | |
| cat "$DRIVER_CONFIG" | jq . | |
| echo "Workload phases:" | |
| jq '.phases[] | {id, completion}' "$WORKLOAD_CONFIG" | |
| - name: Analyze driver requirements | |
| id: driver-info | |
| run: | | |
| DRIVER="${{ steps.validate.outputs.driver }}" | |
| PRIMARY_VERSION="${{ steps.inputs.outputs.primary_version }}" | |
| SECONDARY_VERSION="${{ steps.inputs.outputs.secondary_version }}" | |
| DRIVER_CONFIG="${{ steps.validate.outputs.driver_config }}" | |
| DRIVER_ID=$(jq -r '.driver_id' "$DRIVER_CONFIG") | |
| SECONDARY_DRIVER_ID=$(jq -r '.specific_driver_config.secondary_driver_id // empty' "$DRIVER_CONFIG") | |
| echo "driver_id=$DRIVER_ID" >> $GITHUB_OUTPUT | |
| echo "secondary_driver_id=$SECONDARY_DRIVER_ID" >> $GITHUB_OUTPUT | |
| # Check if primary version is a commit ID (7-40 hex chars) or version number | |
| PRIMARY_VERSION="${{ steps.inputs.outputs.primary_version }}" | |
| BUILD_GLIDE=false | |
| if [ -n "$PRIMARY_VERSION" ] && [[ "$PRIMARY_VERSION" =~ ^[0-9a-fA-F]{7,40}$ ]]; then | |
| echo "primary_is_commit=true" >> $GITHUB_OUTPUT | |
| echo "Detected: primary_version '$PRIMARY_VERSION' is a commit ID" | |
| if [ "$DRIVER_ID" = "spring-data-valkey" ]; then | |
| echo "build_sdv=true" >> $GITHUB_OUTPUT | |
| echo "sdv_version=${PRIMARY_VERSION:0:8}-SNAPSHOT" >> $GITHUB_OUTPUT | |
| elif [ "$DRIVER_ID" = "valkey-glide" ]; then | |
| BUILD_GLIDE=true | |
| else | |
| echo "ERROR: Commit IDs are only supported for spring-data-valkey and valkey-glide primary drivers" | |
| exit 1 | |
| fi | |
| else | |
| echo "primary_is_commit=false" >> $GITHUB_OUTPUT | |
| echo "build_sdv=false" >> $GITHUB_OUTPUT | |
| if [ -n "$PRIMARY_VERSION" ]; then | |
| echo "Detected: primary_version '$PRIMARY_VERSION' is a release version" | |
| fi | |
| fi | |
| # Check if secondary version is a commit ID (7-40 hex chars) or version number | |
| if [ -n "$SECONDARY_DRIVER_ID" ] && [ -n "$SECONDARY_VERSION" ] && [[ "$SECONDARY_VERSION" =~ ^[0-9a-fA-F]{7,40}$ ]]; then | |
| if [ "$SECONDARY_DRIVER_ID" = "valkey-glide" ]; then | |
| echo "secondary_is_commit=true" >> $GITHUB_OUTPUT | |
| echo "Detected: secondary_version '$SECONDARY_VERSION' is a commit ID" | |
| BUILD_GLIDE=true | |
| else | |
| echo "ERROR: Commit IDs are only supported for valkey-glide secondary driver" | |
| exit 1 | |
| fi | |
| else | |
| echo "secondary_is_commit=false" >> $GITHUB_OUTPUT | |
| if [ -n "$SECONDARY_VERSION" ]; then | |
| echo "Detected: secondary_version '$SECONDARY_VERSION' is a release version" | |
| fi | |
| fi | |
| echo "build_glide=$BUILD_GLIDE" >> $GITHUB_OUTPUT | |
| if [ "$BUILD_GLIDE" = "true" ]; then | |
| # Resolve glide commit ID and version — glide can be primary or secondary | |
| if [ "$DRIVER_ID" = "valkey-glide" ] && [[ "$PRIMARY_VERSION" =~ ^[0-9a-fA-F]{7,40}$ ]]; then | |
| GLIDE_COMMIT="$PRIMARY_VERSION" | |
| else | |
| GLIDE_COMMIT="$SECONDARY_VERSION" | |
| fi | |
| echo "glide_commit=$GLIDE_COMMIT" >> $GITHUB_OUTPUT | |
| echo "glide_version=${GLIDE_COMMIT:0:8}-SNAPSHOT" >> $GITHUB_OUTPUT | |
| fi | |
| echo "" | |
| echo "=== Driver Analysis ===" | |
| echo "Primary driver: $DRIVER_ID" | |
| echo "Primary version: ${PRIMARY_VERSION:-<default>}" | |
| echo "Secondary driver: ${SECONDARY_DRIVER_ID:-<none>}" | |
| echo "Secondary version: ${SECONDARY_VERSION:-<default>}" | |
| - name: Validate and resolve versions | |
| id: versions | |
| run: | | |
| PRIMARY_VERSION="${{ steps.inputs.outputs.primary_version }}" | |
| SECONDARY_VERSION="${{ steps.inputs.outputs.secondary_version }}" | |
| DRIVER_ID="${{ steps.driver-info.outputs.driver_id }}" | |
| SECONDARY_DRIVER_ID="${{ steps.driver-info.outputs.secondary_driver_id }}" | |
| # Cannot provide secondary version if driver has no secondary driver | |
| if [ -z "$SECONDARY_DRIVER_ID" ] && [ -n "$SECONDARY_VERSION" ]; then | |
| echo "ERROR: secondary_version provided but driver has no secondary driver" | |
| exit 1 | |
| fi | |
| echo "primary_version=$PRIMARY_VERSION" >> $GITHUB_OUTPUT | |
| echo "secondary_version=$SECONDARY_VERSION" >> $GITHUB_OUTPUT | |
| - name: Cache Valkey server binary | |
| id: cache-valkey | |
| uses: actions/cache@v4 | |
| with: | |
| path: resp-bench/work/valkey | |
| key: valkey-server-${{ steps.resp-bench.outputs.valkey_version }} | |
| - name: Cache Maven repository | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2/repository | |
| key: maven-bench-${{ steps.resp-bench.outputs.commit }} | |
| restore-keys: | | |
| maven-bench- | |
| - name: Cache async-profiler | |
| id: cache-profiler | |
| uses: actions/cache@v4 | |
| with: | |
| path: /opt/async-profiler | |
| key: async-profiler-3.0 | |
| - name: Restore SDV build from cache | |
| if: steps.driver-info.outputs.build_sdv == 'true' | |
| id: cache-sdv | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2/repository/io/valkey/springframework | |
| key: sdv-build-${{ steps.versions.outputs.primary_version }} | |
| - name: Restore glide build from cache | |
| if: steps.driver-info.outputs.build_glide == 'true' | |
| id: cache-glide | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2/repository/io/valkey/valkey-glide | |
| key: glide-build-${{ steps.driver-info.outputs.glide_commit }} | |
| # All background builds run in the same step so `wait` works across PIDs. | |
| # Per-process logs are captured and displayed on failure. | |
| - name: Parallel builds and installs | |
| run: | | |
| set +e | |
| LOG_DIR="/tmp/build-logs" | |
| mkdir -p "$LOG_DIR" | |
| PIDS=() | |
| NAMES=() | |
| # 1. Pre-compile Valkey server (skip if cached) | |
| if [ "${{ steps.cache-valkey.outputs.cache-hit }}" != "true" ]; then | |
| echo "Starting Valkey server compilation..." | |
| ( | |
| set -euo pipefail | |
| cd resp-bench && make "$(pwd)/work/valkey/bin/valkey-server" | |
| ) > "$LOG_DIR/valkey.log" 2>&1 & | |
| PIDS+=($!); NAMES+=("valkey") | |
| else | |
| echo "Valkey server restored from cache" | |
| fi | |
| # 2. Build SDV from source (if needed and cache miss) | |
| if [ "${{ steps.driver-info.outputs.build_sdv }}" = "true" ] && [ "${{ steps.cache-sdv.outputs.cache-hit }}" != "true" ]; then | |
| echo "Starting SDV build from source..." | |
| ( | |
| set -euo pipefail | |
| COMMIT_ID="${{ steps.versions.outputs.primary_version }}" | |
| SDV_VERSION="${{ steps.driver-info.outputs.sdv_version }}" | |
| # Clone to a separate directory so the main workspace (with orchestrator, | |
| # configs, etc.) is preserved — allows building commits older than the benchmark infra | |
| git clone ${{ github.server_url }}/${{ github.repository }}.git /tmp/spring-data-valkey | |
| cd /tmp/spring-data-valkey | |
| git checkout "$COMMIT_ID" | |
| mvn versions:set -DnewVersion="$SDV_VERSION" -DgenerateBackupPoms=false -q | |
| mvn install -DskipTests -Dmaven.compiler.source=17 -Dmaven.compiler.target=17 -Dgpg.skip=true -q | |
| echo "✓ Built spring-data-valkey $SDV_VERSION" | |
| ) > "$LOG_DIR/sdv.log" 2>&1 & | |
| PIDS+=($!); NAMES+=("sdv") | |
| elif [ "${{ steps.driver-info.outputs.build_sdv }}" = "true" ]; then | |
| echo "SDV build restored from cache" | |
| fi | |
| # 3. Build glide from source (if needed and cache miss) | |
| if [ "${{ steps.driver-info.outputs.build_glide }}" = "true" ] && [ "${{ steps.cache-glide.outputs.cache-hit }}" != "true" ]; then | |
| echo "Starting glide build from source..." | |
| ( | |
| set -euo pipefail | |
| # Install Rust | |
| curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y | |
| source "$HOME/.cargo/env" | |
| # Install protobuf compiler v29.1 | |
| PROTOC_VERSION="29.1" | |
| curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip" | |
| unzip -o "protoc-${PROTOC_VERSION}-linux-x86_64.zip" -d "$HOME/.local" | |
| rm "protoc-${PROTOC_VERSION}-linux-x86_64.zip" | |
| export PATH="$PATH:$HOME/.local/bin" | |
| # Install ziglang and cargo-zigbuild (required for Linux builds) | |
| sudo python3 -m pip install ziglang | |
| cargo install --locked cargo-zigbuild | |
| COMMIT_ID="${{ steps.driver-info.outputs.glide_commit }}" | |
| GLIDE_VERSION="${{ steps.driver-info.outputs.glide_version }}" | |
| # Clone valkey-glide at specific commit | |
| git clone https://github.com/valkey-io/valkey-glide.git /tmp/valkey-glide | |
| cd /tmp/valkey-glide | |
| git checkout "$COMMIT_ID" | |
| # build with debug symbols for flamegraph visibility | |
| cd java | |
| CARGO_PROFILE_RELEASE_DEBUG=true ./gradlew :client:buildAll -x test | |
| GLIDE_RELEASE_VERSION="$GLIDE_VERSION" ./gradlew publishToMavenLocal -x test | |
| echo "✓ Built valkey-glide $GLIDE_VERSION" | |
| ) > "$LOG_DIR/glide.log" 2>&1 & | |
| PIDS+=($!); NAMES+=("glide") | |
| elif [ "${{ steps.driver-info.outputs.build_glide }}" = "true" ]; then | |
| echo "Glide build restored from cache" | |
| fi | |
| # 4. Install Python deps | |
| echo "Starting pip install..." | |
| ( | |
| set -euo pipefail | |
| sudo python3 -m pip install psycopg2-binary boto3 hdrhistogram | |
| ) > "$LOG_DIR/pip.log" 2>&1 & | |
| PIDS+=($!); NAMES+=("pip") | |
| # 5. Install async-profiler (skip if cached) | |
| if [ "${{ steps.cache-profiler.outputs.cache-hit }}" != "true" ]; then | |
| echo "Downloading async-profiler..." | |
| ( | |
| set -euo pipefail | |
| curl -sL "https://github.com/async-profiler/async-profiler/releases/download/v3.0/async-profiler-3.0-linux-x64.tar.gz" | sudo tar xzf - -C /opt | |
| sudo mv /opt/async-profiler-3.0-linux-x64 /opt/async-profiler | |
| ) > "$LOG_DIR/ap.log" 2>&1 & | |
| PIDS+=($!); NAMES+=("ap") | |
| else | |
| echo "async-profiler restored from cache" | |
| fi | |
| # Wait for all background processes | |
| echo "" | |
| echo "=== Waiting for background builds ===" | |
| FAILED=0 | |
| for i in "${!PIDS[@]}"; do | |
| echo -n " ${NAMES[$i]}: " | |
| if wait "${PIDS[$i]}"; then | |
| echo "OK" | |
| else | |
| EXIT_CODE=$? | |
| echo "FAILED (exit $EXIT_CODE)" | |
| echo "::error::${NAMES[$i]} build failed" | |
| echo "--- ${NAMES[$i]} log ---" | |
| cat "$LOG_DIR/${NAMES[$i]}.log" 2>/dev/null || echo "(no log)" | |
| echo "--- end ---" | |
| FAILED=1 | |
| fi | |
| done | |
| if [ "$FAILED" -ne 0 ]; then | |
| exit 1 | |
| fi | |
| echo "" | |
| echo "=== All builds completed ===" | |
| resp-bench/work/valkey/bin/valkey-server --version | |
| /opt/async-profiler/bin/asprof --version | |
| command -v perf && perf --version || echo "perf not available" | |
| - name: Update resp-bench driver versions | |
| run: | | |
| cd resp-bench | |
| DRIVER_ID="${{ steps.driver-info.outputs.driver_id }}" | |
| SECONDARY_DRIVER_ID="${{ steps.driver-info.outputs.secondary_driver_id }}" | |
| SECONDARY_VERSION="${{ steps.versions.outputs.secondary_version }}" | |
| SECONDARY_IS_COMMIT="${{ steps.driver-info.outputs.secondary_is_commit }}" | |
| # Update primary driver version if supplied | |
| PRIMARY_VERSION="${{ steps.versions.outputs.primary_version }}" | |
| PRIMARY_IS_COMMIT="${{ steps.driver-info.outputs.primary_is_commit }}" | |
| if [ -n "$PRIMARY_VERSION" ]; then | |
| case "$DRIVER_ID" in | |
| spring-data-valkey) | |
| if [ "$PRIMARY_IS_COMMIT" = "true" ]; then | |
| SDV_VERSION="${{ steps.driver-info.outputs.sdv_version }}" | |
| echo "Updating spring-data-valkey to locally built $SDV_VERSION" | |
| sed -i "s|<spring-data-valkey.version>.*</spring-data-valkey.version>|<spring-data-valkey.version>$SDV_VERSION</spring-data-valkey.version>|" java/pom.xml | |
| else | |
| echo "Updating spring-data-valkey to release version $PRIMARY_VERSION" | |
| sed -i "s|<spring-data-valkey.version>.*</spring-data-valkey.version>|<spring-data-valkey.version>$PRIMARY_VERSION</spring-data-valkey.version>|" java/pom.xml | |
| fi | |
| ;; | |
| valkey-glide) | |
| if [ "$PRIMARY_IS_COMMIT" = "true" ]; then | |
| GLIDE_VERSION="${{ steps.driver-info.outputs.glide_version }}" | |
| echo "Updating valkey-glide (primary) to locally built $GLIDE_VERSION" | |
| sed -i "s|<valkey-glide.version>.*</valkey-glide.version>|<valkey-glide.version>$GLIDE_VERSION</valkey-glide.version>|" java/pom.xml | |
| else | |
| echo "Updating valkey-glide (primary) to release version $PRIMARY_VERSION" | |
| sed -i "s|<valkey-glide.version>.*</valkey-glide.version>|<valkey-glide.version>$PRIMARY_VERSION</valkey-glide.version>|" java/pom.xml | |
| fi | |
| ;; | |
| jedis) | |
| echo "Updating jedis (primary) to $PRIMARY_VERSION" | |
| sed -i "s|<jedis.version>.*</jedis.version>|<jedis.version>$PRIMARY_VERSION</jedis.version>|" java/pom.xml | |
| ;; | |
| lettuce) | |
| echo "Updating lettuce (primary) to $PRIMARY_VERSION" | |
| sed -i "s|<lettuce.version>.*</lettuce.version>|<lettuce.version>$PRIMARY_VERSION</lettuce.version>|" java/pom.xml | |
| ;; | |
| redisson) | |
| echo "Updating redisson (primary) to $PRIMARY_VERSION" | |
| sed -i "s|<redisson.version>.*</redisson.version>|<redisson.version>$PRIMARY_VERSION</redisson.version>|" java/pom.xml | |
| ;; | |
| esac | |
| fi | |
| # Update secondary driver version if a version/commit-id for it was supplied. Otherwise use defaults. | |
| # If secondary version is glide and it was built locally, update the pom.xml file to use this built version. | |
| # Otherwise, update the version supplied in the pom.xml. | |
| if [ -n "$SECONDARY_DRIVER_ID" ] && [ -n "$SECONDARY_VERSION" ]; then | |
| case "$SECONDARY_DRIVER_ID" in | |
| valkey-glide) | |
| if [ "$SECONDARY_IS_COMMIT" = "true" ]; then | |
| # Use the version from the local build | |
| GLIDE_VERSION="${{ steps.driver-info.outputs.glide_version }}" | |
| echo "Updating valkey-glide (secondary) to locally built $GLIDE_VERSION" | |
| sed -i "s|<valkey-glide.version>.*</valkey-glide.version>|<valkey-glide.version>$GLIDE_VERSION</valkey-glide.version>|" java/pom.xml | |
| else | |
| echo "Updating valkey-glide (secondary) to release version $SECONDARY_VERSION" | |
| sed -i "s|<valkey-glide.version>.*</valkey-glide.version>|<valkey-glide.version>$SECONDARY_VERSION</valkey-glide.version>|" java/pom.xml | |
| fi | |
| ;; | |
| jedis) | |
| echo "Updating jedis to $SECONDARY_VERSION" | |
| sed -i "s|<jedis.version>.*</jedis.version>|<jedis.version>$SECONDARY_VERSION</jedis.version>|" java/pom.xml | |
| ;; | |
| lettuce) | |
| echo "Updating lettuce to $SECONDARY_VERSION" | |
| sed -i "s|<lettuce.version>.*</lettuce.version>|<lettuce.version>$SECONDARY_VERSION</lettuce.version>|" java/pom.xml | |
| ;; | |
| esac | |
| fi | |
| echo "" | |
| echo "=== Updated pom.xml versions ===" | |
| grep -E "(spring-data-valkey|valkey-glide|jedis|lettuce)\.version" java/pom.xml | |
| - name: Build resp-bench Java benchmark | |
| run: | | |
| cd resp-bench | |
| make java-build | |
| echo "✓ Java benchmark built" | |
| ls -la java/target/*.jar | |
| - name: Run benchmark and publish results to DB | |
| run: | | |
| DRIVER="${{ steps.validate.outputs.driver }}" | |
| WORKLOAD="${{ steps.validate.outputs.workload }}" | |
| DRIVER_CONFIG="${{ steps.validate.outputs.driver_config }}" | |
| WORKLOAD_CONFIG="${{ steps.validate.outputs.workload_config }}" | |
| JOB_ID_PREFIX="${{ steps.inputs.outputs.job_id_prefix }}" | |
| PREFIX_ARG="" | |
| [ -n "$JOB_ID_PREFIX" ] && PREFIX_ARG="--job-id-prefix $JOB_ID_PREFIX" | |
| NETWORK_DELAY="${{ steps.inputs.outputs.network_delay }}" | |
| NETWORK_JITTER="${{ steps.inputs.outputs.network_jitter }}" | |
| NETWORK_DELAY_DISTRIBUTION="${{ steps.inputs.outputs.network_delay_distribution }}" | |
| TC_ARGS="" | |
| [ -n "$NETWORK_DELAY" ] && TC_ARGS="$TC_ARGS --network-delay $NETWORK_DELAY" | |
| [ -n "$NETWORK_JITTER" ] && TC_ARGS="$TC_ARGS --network-jitter $NETWORK_JITTER" | |
| [ -n "$NETWORK_DELAY_DISTRIBUTION" ] && TC_ARGS="$TC_ARGS --network-delay-distribution $NETWORK_DELAY_DISTRIBUTION" | |
| python3 -u .github/workflows/benchmark_orchestrator.py \ | |
| --output "benchmark_results_${DRIVER}_${WORKLOAD}.json" \ | |
| --workload-config "$WORKLOAD_CONFIG" \ | |
| --driver-config "$DRIVER_CONFIG" \ | |
| --resp-bench-dir "${{ github.workspace }}/resp-bench" \ | |
| --resp-bench-commit "${{ steps.resp-bench.outputs.commit }}" \ | |
| --s3-bucket "${{ secrets.BENCHMARK_S3_BUCKET }}" \ | |
| --pg-host "${{ secrets.BENCHMARK_PG_HOST }}" \ | |
| --pg-secret-name "${{ secrets.BENCHMARK_PG_SECRET_NAME }}" \ | |
| $PREFIX_ARG $TC_ARGS | |
| - name: Display results summary | |
| if: always() | |
| run: | | |
| DRIVER="${{ steps.validate.outputs.driver }}" | |
| WORKLOAD="${{ steps.validate.outputs.workload }}" | |
| RESULT_FILE="benchmark_results_${DRIVER}_${WORKLOAD}.json" | |
| if [ -f "$RESULT_FILE" ]; then | |
| echo "=== Benchmark Results Summary ===" | |
| python3 -c " | |
| import json | |
| with open('$RESULT_FILE') as f: | |
| data = json.load(f) | |
| print(f\"Job ID: {data['job_id']}\") | |
| print(f\"Driver: {data['config']['driver']['driver_id']}\") | |
| print(f\"Workload: {data['config']['workload']['benchmark_profile']['name']}\") | |
| print(f\"Elapsed: {data['results']['elapsed_ms']}ms\") | |
| print() | |
| versions = data.get('versions', {}) | |
| print(f\"Primary: {versions.get('primary_driver_id')} @ {versions.get('primary_driver_version', 'N/A')}\") | |
| print(f\"Secondary: {versions.get('secondary_driver_id', 'N/A')} @ {versions.get('secondary_driver_version', 'N/A')}\") | |
| if versions.get('commit_id'): | |
| print(f\"resp-bench: {versions.get('commit_id')}\") | |
| print() | |
| steady = data['results']['phases'].get('STEADY', {}) | |
| if steady: | |
| totals = steady.get('totals', {}) | |
| print(f\"STEADY totals: {totals.get('requests', 0):,} requests, {totals.get('errors', 0)} errors\") | |
| print() | |
| print('Per-command metrics:') | |
| for cmd, m in steady.get('metrics', {}).items(): | |
| reqs = m['requests'] | |
| errs = m['errors'] | |
| lat = m.get('latency', {}) | |
| summary = lat.get('summary', {}) | |
| print(f\" {cmd}:\") | |
| print(f\" Requests: {reqs:,} (errors: {errs})\") | |
| print(f\" Latency: min={summary.get('min')} p50={summary.get('p50')} p95={summary.get('p95')} p99={summary.get('p99')} p999={summary.get('p999')} max={summary.get('max')} us\") | |
| print() | |
| print('Perf:') | |
| perf = data['results']['perf']['counters'] | |
| if perf.get('ipc'): | |
| print(f\" IPC: {perf['ipc']}\") | |
| if perf.get('cache_miss_rate'): | |
| print(f\" Cache miss rate: {perf['cache_miss_rate']}%\") | |
| if perf.get('branch_miss_rate'): | |
| print(f\" Branch miss rate: {perf['branch_miss_rate']}%\") | |
| " | |
| else | |
| echo "⚠ Result file not found: $RESULT_FILE" | |
| fi | |
| - name: Generate summary report | |
| if: always() | |
| run: | | |
| DRIVER="${{ steps.validate.outputs.driver }}" | |
| WORKLOAD="${{ steps.validate.outputs.workload }}" | |
| RESULT_FILE="benchmark_results_${DRIVER}_${WORKLOAD}.json" | |
| echo "# Benchmark Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ -f "$RESULT_FILE" ]; then | |
| python3 << EOF >> $GITHUB_STEP_SUMMARY | |
| import json | |
| with open("$RESULT_FILE") as f: | |
| data = json.load(f) | |
| job_id = data["job_id"] | |
| versions = data.get("versions", {}) | |
| driver = data["config"]["driver"]["driver_id"] | |
| workload_profile = data["config"]["workload"]["benchmark_profile"]["name"] | |
| elapsed = data["results"]["elapsed_ms"] | |
| network_delay = data["results"].get("network_delay", "") | |
| network_jitter = data["results"].get("network_jitter", "") | |
| network_distribution = data["results"].get("network_delay_distribution", "") | |
| pv = versions.get("primary_driver_version") or "N/A" | |
| sv = versions.get("secondary_driver_version") | |
| sid = versions.get("secondary_driver_id") | |
| commit_id = versions.get("commit_id") | |
| print(f"**Job ID:** \`{job_id}\`") | |
| print(f"**Driver:** {driver} @ \`{pv}\`") | |
| if sid and sv: | |
| print(f"**Secondary:** {sid} @ \`{sv}\`") | |
| if commit_id: | |
| print(f"**resp-bench:** \`{commit_id}\`") | |
| print(f"**Workload config:** \`${WORKLOAD}\` ({workload_profile})") | |
| print(f"**Driver config:** \`${DRIVER}\`") | |
| delay_str = network_delay or "disabled" | |
| if network_jitter: | |
| delay_str += f" jitter {network_jitter}" | |
| if network_distribution: | |
| delay_str += f" distribution {network_distribution}" | |
| print(f"**Simulated network delay:** {delay_str}") | |
| print(f"**Elapsed:** {elapsed}ms") | |
| print() | |
| print("| Versions | Driver | Workload | Status | Elapsed |") | |
| print("|----------|--------|----------|--------|---------|") | |
| version_str = pv | |
| if sv: | |
| version_str += f" ({sid}: {sv})" | |
| print(f"| {version_str} | {driver} | \`${WORKLOAD}\` | ✅ | {elapsed}ms |") | |
| print() | |
| steady = data["results"]["phases"].get("STEADY", {}) | |
| if steady: | |
| print("### Steady State Metrics") | |
| print("| Command | Requests | Errors | p50 (μs) | p95 (μs) | p99 (μs) | p999 (μs) |") | |
| print("|---------|----------|--------|----------|----------|----------|-----------|") | |
| for cmd, m in steady.get("metrics", {}).items(): | |
| s = m.get("latency", {}).get("summary", {}) | |
| print(f"| {cmd} | {m['requests']:,} | {m['errors']} | {s.get('p50','-')} | {s.get('p95','-')} | {s.get('p99','-')} | {s.get('p999','-')} |") | |
| EOF | |
| else | |
| echo "❌ No results found" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Upload benchmark results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results-${{ steps.validate.outputs.driver }}-${{ steps.validate.outputs.workload }} | |
| path: | | |
| benchmark_results_${{ steps.validate.outputs.driver }}_${{ steps.validate.outputs.workload }}.json | |
| benchmark_results_${{ steps.validate.outputs.driver }}_${{ steps.validate.outputs.workload }}.ndjson | |
| benchmark_results_${{ steps.validate.outputs.driver }}_${{ steps.validate.outputs.workload }}_collapsed.txt |