Benchmark: Add json config files and support custom json config as CI input

.github/workflows/benchmark.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       primary_driver:
-        description: "Primary driver (for details, see resp-bench repo)"
+        description: "Primary driver"
         required: true
         type: choice
         options:
@@ -26,13 +26,21 @@ on:
           - none
         default: "valkey-glide"
       workload:
-        description: "Workload config (for details, see resp-bench repo)"
+        description: "Workload config"
         required: true
         type: choice
         options:
           - example-workload
           - example-workload-single-client
         default: "example-workload"
+      topology:
+        description: "Server topology"
+        required: true
+        type: choice
+        options:
+          - standalone
+          - cluster
+        default: "standalone"
       primary_version:
         description: "Primary driver version. Leave empty for spring-data-valkey (uses branch HEAD)."
         required: false
@@ -48,6 +56,16 @@ on:
         required: false
         type: string
         default: ""
+      custom_driver_config:
+        description: "Custom driver config JSON (overrides driver/topology selection)"
+        required: false
+        type: string
+        default: ""
+      custom_workload_config:
+        description: "Custom workload config JSON (overrides workload selection)"
+        required: false
+        type: string
+        default: ""
 
 env:
   RESP_BENCH_REPO: "https://github.com/ikolomi/resp-bench.git"
@@ -82,13 +100,15 @@ jobs:
           PRIMARY_DRIVER="${{ github.event.inputs.primary_driver }}"
           SECONDARY_DRIVER="${{ github.event.inputs.secondary_driver }}"
           WORKLOAD="${{ github.event.inputs.workload }}"
+          TOPOLOGY="${{ github.event.inputs.topology }}"
           PRIMARY_VERSION="${{ github.event.inputs.primary_version }}"
           SECONDARY_VERSION="${{ github.event.inputs.secondary_version }}"
           JOB_ID_PREFIX="${{ github.event.inputs.job_id_prefix }}"
 
           echo "primary_driver=$PRIMARY_DRIVER" >> $GITHUB_OUTPUT
           echo "secondary_driver=$SECONDARY_DRIVER" >> $GITHUB_OUTPUT
           echo "workload=$WORKLOAD" >> $GITHUB_OUTPUT
+          echo "topology=$TOPOLOGY" >> $GITHUB_OUTPUT
           echo "primary_version=$PRIMARY_VERSION" >> $GITHUB_OUTPUT
           echo "secondary_version=$SECONDARY_VERSION" >> $GITHUB_OUTPUT
           echo "job_id_prefix=$JOB_ID_PREFIX" >> $GITHUB_OUTPUT
@@ -97,6 +117,7 @@ jobs:
           echo "Primary driver:    $PRIMARY_DRIVER"
           echo "Secondary driver:  $SECONDARY_DRIVER"
           echo "Workload:          $WORKLOAD"
+          echo "Topology:          $TOPOLOGY"
           echo "Primary version:   '${PRIMARY_VERSION}'"
           echo "Secondary version: '${SECONDARY_VERSION}'"
           echo "Job ID prefix:     '${JOB_ID_PREFIX}'"
@@ -108,53 +129,94 @@ jobs:
           PRIMARY_DRIVER="${{ steps.inputs.outputs.primary_driver }}"
           SECONDARY_DRIVER="${{ steps.inputs.outputs.secondary_driver }}"
           WORKLOAD="${{ steps.inputs.outputs.workload }}"
-
-          # Resolve driver config filename based on primary and secondary driver
-          if [[ "$PRIMARY_DRIVER" == spring-data-* ]]; then
-            # spring-data-* drivers require a secondary driver
-            if [ "$SECONDARY_DRIVER" = "none" ] || [ -z "$SECONDARY_DRIVER" ]; then
-              echo "ERROR: $PRIMARY_DRIVER requires a secondary driver (valkey-glide, jedis, or lettuce)"
-              exit 1
-            fi
-            # spring-data-redis cannot use valkey-glide
-            if [ "$PRIMARY_DRIVER" = "spring-data-redis" ] && [ "$SECONDARY_DRIVER" = "valkey-glide" ]; then
-              echo "ERROR: spring-data-redis does not support valkey-glide. Use jedis or lettuce instead."
+          TOPOLOGY="${{ steps.inputs.outputs.topology }}"
+          CONFIG_DIR=".github/workflows/benchmark_configs"
+          CUSTOM_CONFIG_DIR="/tmp/custom_configs"
+          mkdir -p "$CUSTOM_CONFIG_DIR"
+
+          # Handle custom driver config
+          CUSTOM_DRIVER_CONFIG_INPUT='${{ github.event.inputs.custom_driver_config }}'
+          if [ -n "$CUSTOM_DRIVER_CONFIG_INPUT" ]; then
+            echo "Using custom driver config..."
+            DRIVER_CONFIG="$CUSTOM_CONFIG_DIR/custom-driver.json"
+            printf '%s\n' "$CUSTOM_DRIVER_CONFIG_INPUT" > "$DRIVER_CONFIG"
+            # Validate JSON
+            if ! jq . "$DRIVER_CONFIG" > /dev/null 2>&1; then
+              echo "ERROR: Custom driver config is not valid JSON"
+              echo "Content written:"
+              cat "$DRIVER_CONFIG"
               exit 1
             fi
-            # Map secondary driver name to filename convention (valkey-glide -> glide)
-            SECONDARY_IN_FILENAME="$SECONDARY_DRIVER"
-            if [ "$SECONDARY_DRIVER" = "valkey-glide" ]; then
-              SECONDARY_IN_FILENAME="glide"
-            fi
-            DRIVER="example-${PRIMARY_DRIVER}-${SECONDARY_IN_FILENAME}-standalone"
+            DRIVER="custom-driver"
+            echo "✓ Custom driver config validated"
           else
-            # Standalone drivers (valkey-glide, jedis, lettuce, redisson)
-            DRIVER="example-${PRIMARY_DRIVER}-standalone"
-            if [ "$SECONDARY_DRIVER" != "none" ] && [ -n "$SECONDARY_DRIVER" ]; then
-              echo "Note: secondary_driver '$SECONDARY_DRIVER' ignored for standalone driver '$PRIMARY_DRIVER'"
+            # Resolve driver config filename based on primary and secondary driver
+            if [[ "$PRIMARY_DRIVER" == spring-data-* ]]; then
+              # spring-data-* drivers require a secondary driver
+              if [ "$SECONDARY_DRIVER" = "none" ] || [ -z "$SECONDARY_DRIVER" ]; then
+                echo "ERROR: $PRIMARY_DRIVER requires a secondary driver (valkey-glide, jedis, or lettuce)"
+                exit 1
+              fi
+              # spring-data-redis cannot use valkey-glide
+              if [ "$PRIMARY_DRIVER" = "spring-data-redis" ] && [ "$SECONDARY_DRIVER" = "valkey-glide" ]; then
+                echo "ERROR: spring-data-redis does not support valkey-glide. Use jedis or lettuce instead."
+                exit 1
+              fi
+              # Map secondary driver name to filename convention (valkey-glide -> glide)
+              SECONDARY_IN_FILENAME="$SECONDARY_DRIVER"
+              if [ "$SECONDARY_DRIVER" = "valkey-glide" ]; then
+                SECONDARY_IN_FILENAME="glide"
+              fi
+              DRIVER="example-${PRIMARY_DRIVER}-${SECONDARY_IN_FILENAME}-${TOPOLOGY}"
+            else
+              # Standalone drivers (valkey-glide, jedis, lettuce, redisson)
+              DRIVER="example-${PRIMARY_DRIVER}-${TOPOLOGY}"
+              if [ "$SECONDARY_DRIVER" != "none" ] && [ -n "$SECONDARY_DRIVER" ]; then
+                echo "Note: secondary_driver '$SECONDARY_DRIVER' ignored for standalone driver '$PRIMARY_DRIVER'"
+              fi
             fi
-          fi
+            DRIVER_CONFIG="${CONFIG_DIR}/drivers/${DRIVER}.json"
 
-          DRIVER_CONFIG="resp-bench/configs/drivers/${DRIVER}.json"
-          WORKLOAD_CONFIG="resp-bench/configs/workloads/${WORKLOAD}.json"
-
-          if [ ! -f "$DRIVER_CONFIG" ]; then
-            echo "ERROR: Driver config not found: $DRIVER_CONFIG"
-            echo "Available driver configs:"
-            ls -la resp-bench/configs/drivers/
-            exit 1
+            if [ ! -f "$DRIVER_CONFIG" ]; then
+              echo "ERROR: Driver config not found: $DRIVER_CONFIG"
+              echo "Available driver configs:"
+              ls -la "${CONFIG_DIR}/drivers/"
+              exit 1
+            fi
+            echo "✓ Driver config: $DRIVER_CONFIG"
           fi
-          echo "✓ Driver config: $DRIVER_CONFIG"
 
-          if [ ! -f "$WORKLOAD_CONFIG" ]; then
-            echo "ERROR: Workload config not found: $WORKLOAD_CONFIG"
-            echo "Available workload configs:"
-            ls -la resp-bench/configs/workloads/
-            exit 1
+          # Handle custom workload config
+          CUSTOM_WORKLOAD_CONFIG_INPUT='${{ github.event.inputs.custom_workload_config }}'
+          if [ -n "$CUSTOM_WORKLOAD_CONFIG_INPUT" ]; then
+            echo "Using custom workload config..."
+            WORKLOAD_CONFIG="$CUSTOM_CONFIG_DIR/custom-workload.json"
+            printf '%s\n' "$CUSTOM_WORKLOAD_CONFIG_INPUT" > "$WORKLOAD_CONFIG"
+            # Validate JSON
+            if ! jq . "$WORKLOAD_CONFIG" > /dev/null 2>&1; then
+              echo "ERROR: Custom workload config is not valid JSON"
+              echo "Content written:"
+              cat "$WORKLOAD_CONFIG"
+              exit 1
+            fi
+            WORKLOAD="custom-workload"
+            echo "✓ Custom workload config validated"
+          else
+            WORKLOAD_CONFIG="${CONFIG_DIR}/workloads/${WORKLOAD}.json"
+
+            if [ ! -f "$WORKLOAD_CONFIG" ]; then
+              echo "ERROR: Workload config not found: $WORKLOAD_CONFIG"
+              echo "Available workload configs:"
+              ls -la "${CONFIG_DIR}/workloads/"
+              exit 1
+            fi
+            echo "✓ Workload config: $WORKLOAD_CONFIG"
           fi
-          echo "✓ Workload config: $WORKLOAD_CONFIG"
 
           echo "driver=$DRIVER" >> $GITHUB_OUTPUT
+          echo "driver_config=$DRIVER_CONFIG" >> $GITHUB_OUTPUT
+          echo "workload=$WORKLOAD" >> $GITHUB_OUTPUT
+          echo "workload_config=$WORKLOAD_CONFIG" >> $GITHUB_OUTPUT
 
           echo "Driver config contents:"
           cat "$DRIVER_CONFIG" | jq .
@@ -167,7 +229,7 @@ jobs:
         run: |
           DRIVER="${{ steps.validate.outputs.driver }}"
           SECONDARY_VERSION="${{ steps.inputs.outputs.secondary_version }}"
-          DRIVER_CONFIG="resp-bench/configs/drivers/${DRIVER}.json"
+          DRIVER_CONFIG="${{ steps.validate.outputs.driver_config }}"
 
           DRIVER_ID=$(jq -r '.driver_id' "$DRIVER_CONFIG")
           SECONDARY_DRIVER_ID=$(jq -r '.specific_driver_config.secondary_driver_id // empty' "$DRIVER_CONFIG")
@@ -384,16 +446,18 @@ jobs:
       - name: Run benchmark and publish results to DB
         run: |
           DRIVER="${{ steps.validate.outputs.driver }}"
-          WORKLOAD="${{ steps.inputs.outputs.workload }}"
+          WORKLOAD="${{ steps.validate.outputs.workload }}"
+          DRIVER_CONFIG="${{ steps.validate.outputs.driver_config }}"
+          WORKLOAD_CONFIG="${{ steps.validate.outputs.workload_config }}"
           JOB_ID_PREFIX="${{ steps.inputs.outputs.job_id_prefix }}"
 
           PREFIX_ARG=""
           [ -n "$JOB_ID_PREFIX" ] && PREFIX_ARG="--job-id-prefix $JOB_ID_PREFIX"
 
           python3 -u .github/workflows/benchmark_orchestrator.py \
             --output "benchmark_results_${DRIVER}_${WORKLOAD}.json" \
-            --workload-config "resp-bench/configs/workloads/${WORKLOAD}.json" \
-            --driver-config "resp-bench/configs/drivers/${DRIVER}.json" \
+            --workload-config "$WORKLOAD_CONFIG" \
+            --driver-config "$DRIVER_CONFIG" \
             --resp-bench-dir "${{ github.workspace }}/resp-bench" \
             --resp-bench-commit "${{ steps.resp-bench.outputs.commit }}" \
             --s3-bucket "${{ secrets.BENCHMARK_S3_BUCKET }}" \
@@ -405,7 +469,7 @@ jobs:
         if: always()
         run: |
           DRIVER="${{ steps.validate.outputs.driver }}"
-          WORKLOAD="${{ steps.inputs.outputs.workload }}"
+          WORKLOAD="${{ steps.validate.outputs.workload }}"
           RESULT_FILE="benchmark_results_${DRIVER}_${WORKLOAD}.json"
 
           if [ -f "$RESULT_FILE" ]; then
@@ -461,7 +525,7 @@ jobs:
         if: always()
         run: |
           DRIVER="${{ steps.validate.outputs.driver }}"
-          WORKLOAD="${{ steps.inputs.outputs.workload }}"
+          WORKLOAD="${{ steps.validate.outputs.workload }}"
           RESULT_FILE="benchmark_results_${DRIVER}_${WORKLOAD}.json"
 
           echo "# Benchmark Results" >> $GITHUB_STEP_SUMMARY
@@ -522,8 +586,8 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: benchmark-results-${{ steps.validate.outputs.driver }}-${{ steps.inputs.outputs.workload }}
+          name: benchmark-results-${{ steps.validate.outputs.driver }}-${{ steps.validate.outputs.workload }}
           path: |
-            benchmark_results_${{ steps.validate.outputs.driver }}_${{ steps.inputs.outputs.workload }}.json
-            benchmark_results_${{ steps.validate.outputs.driver }}_${{ steps.inputs.outputs.workload }}.ndjson
-            benchmark_results_${{ steps.validate.outputs.driver }}_${{ steps.inputs.outputs.workload }}_collapsed.txt
+            benchmark_results_${{ steps.validate.outputs.driver }}_${{ steps.validate.outputs.workload }}.json
+            benchmark_results_${{ steps.validate.outputs.driver }}_${{ steps.validate.outputs.workload }}.ndjson
+            benchmark_results_${{ steps.validate.outputs.driver }}_${{ steps.validate.outputs.workload }}_collapsed.txt

.github/workflows/benchmark_configs/README.md

@@ -0,0 +1,81 @@
+# Benchmark Configuration Guide
+
+This directory contains configuration files for the benchmark CI workflow.
+
+## CI Workflow Inputs
+
+### Predefined Configs
+
+| Input              | Description                          | Options                                                                                   |
+| ------------------ | ------------------------------------ | ----------------------------------------------------------------------------------------- |
+| `primary_driver`   | Client library to benchmark          | `spring-data-valkey`, `spring-data-redis`, `valkey-glide`, `jedis`, `lettuce`, `redisson` |
+| `secondary_driver` | Underlying driver for spring-data-\* | `valkey-glide`, `jedis`, `lettuce`, `none`                                                |
+| `topology`         | Server topology                      | `standalone`, `cluster`                                                                   |
+| `workload`         | Test scenario                        | `example-workload`, `example-workload-single-client`                                      |
+
+### Custom Configs
+
+For custom use cases, you can provide custom JSON configs directly:
+
+| Input                    | Description                                              |
+| ------------------------ | -------------------------------------------------------- |
+| `custom_driver_config`   | Custom driver JSON (overrides driver/topology selection) |
+| `custom_workload_config` | Custom workload JSON (overrides workload selection)      |
+
+### Version Inputs
+
+| Input               | Description                                                                                                 |
+| ------------------- | ----------------------------------------------------------------------------------------------------------- |
+| `primary_version`   | Version of primary driver. Leave empty for `spring-data-valkey` (uses branch HEAD)                          |
+| `secondary_version` | Version or commit ID of secondary driver. Commit IDs (7-40 hex chars) trigger source build for valkey-glide |
+| `job_id_prefix`     | Optional prefix for job ID (e.g., `nightly`, `pr-123`)                                                      |
+
+## How Config Selection Works
+
+The CI workflow maps your input selections to JSON config files using a naming convention.
+
+### Driver Config Resolution
+
+The workflow constructs the driver config filename from your inputs:
+
+| Primary Driver       | Secondary Driver | Topology     | Resolved File                                      |
+| -------------------- | ---------------- | ------------ | -------------------------------------------------- |
+| `valkey-glide`       | (ignored)        | `standalone` | `example-valkey-glide-standalone.json`             |
+| `valkey-glide`       | (ignored)        | `cluster`    | `example-valkey-glide-cluster.json`                |
+| `spring-data-valkey` | `valkey-glide`   | `standalone` | `example-spring-data-valkey-glide-standalone.json` |
+| `spring-data-valkey` | `valkey-glide`   | `cluster`    | `example-spring-data-valkey-glide-cluster.json`    |
+| `spring-data-valkey` | `jedis`          | `cluster`    | `example-spring-data-valkey-jedis-cluster.json`    |
+
+**Pattern:**
+
+- Standalone drivers: `example-{primary_driver}-{topology}.json`
+- Spring-data drivers: `example-{primary_driver}-{secondary_driver}-{topology}.json`
+  - Note: `valkey-glide` is shortened to `glide` in filenames
+
+### Workload Config Resolution
+
+The `workload` input maps directly to a file:
+
+- `example-workload` → `workloads/example-workload.json`
+- `example-workload-single-client` → `workloads/example-workload-single-client.json`
+
+### Custom Configs
+
+When you provide `custom_driver_config` or `custom_workload_config`, the workflow:
+
+1. Writes your JSON to a temp file
+2. Validates it with `jq`
+3. Uses it instead of the predefined config
+
+This allows testing configurations not in the predefined set.
+
+For the full specification of config formats to use when constructing custom configs, see the JSON schema files:
+
+- **Driver config schema:** [schemas/driver-config.schema.json](schemas/driver-config.schema.json)
+- **Workload config schema:** [schemas/workload-config.schema.json](schemas/workload-config.schema.json)
+
+### Key config rules:
+
+- `spring-data-valkey` and `spring-data-redis` require `secondary_driver_id`
+- `spring-data-redis` does NOT support `valkey-glide` as secondary driver
+- Standalone drivers (`jedis`, `lettuce`, `valkey-glide`, `redisson`) ignore `secondary_driver_id`

.github/workflows/benchmark_configs/drivers/example-jedis-cluster.json

@@ -0,0 +1,7 @@
+{
+  "schema_version": "1.0",
+  "description": "Jedis client - cluster mode",
+  "driver_id": "jedis",
+  "mode": "cluster",
+  "specific_driver_config": {}
+}

.github/workflows/benchmark_configs/drivers/example-jedis-standalone.json

@@ -0,0 +1,7 @@
+{
+  "schema_version": "1.0",
+  "description": "Jedis client - standalone mode",
+  "driver_id": "jedis",
+  "mode": "standalone",
+  "specific_driver_config": {}
+}

.github/workflows/benchmark_configs/drivers/example-lettuce-cluster.json

@@ -0,0 +1,7 @@
+{
+  "schema_version": "1.0",
+  "description": "Lettuce client - cluster mode",
+  "driver_id": "lettuce",
+  "mode": "cluster",
+  "specific_driver_config": {}
+}

.github/workflows/benchmark_configs/drivers/example-lettuce-standalone.json

@@ -0,0 +1,7 @@
+{
+  "schema_version": "1.0",
+  "description": "Lettuce client - standalone mode",
+  "driver_id": "lettuce",
+  "mode": "standalone",
+  "specific_driver_config": {}
+}

.github/workflows/benchmark_configs/drivers/example-redisson-cluster.json

@@ -0,0 +1,7 @@
+{
+  "schema_version": "1.0",
+  "description": "Redisson client - cluster mode",
+  "driver_id": "redisson",
+  "mode": "cluster",
+  "specific_driver_config": {}
+}

.github/workflows/benchmark_configs/drivers/example-redisson-standalone.json

@@ -0,0 +1,7 @@
+{
+  "schema_version": "1.0",
+  "description": "Redisson client - standalone mode",
+  "driver_id": "redisson",
+  "mode": "standalone",
+  "specific_driver_config": {}
+}