feat: Add custom_image input to debug Spice Cloud workflow

Add a custom_image workflow input to run_spicebench_debug_spice_cloud
that allows specifying a custom runtime container image (e.g.
ghcr.io/spiceai/spiceai-dev:spicebench-sf10) instead of the default
nightly image.

When set, the image reference is parsed into registry, image name, and
tag components and passed through to spidapter as SPIDAPTER_IMAGE_REGISTRY,
SPIDAPTER_IMAGE_NAME, and SPIDAPTER_IMAGE_TAG env vars. The channel is
automatically switched to internal.

Also adds executor_memory_limit input and fixes NUM_QUERY_CLIENTS to
match the main workflow (2 instead of 8).

Author: phillipleblanc

.github/workflows/run_spicebench_debug_spice_cloud.yml

@@ -46,6 +46,11 @@ on:
         required: false
         default: 'latest'
         type: string
+      num_query_clients:
+        description: 'Number of concurrent query clients'
+        required: false
+        default: '2'
+        type: string
       enable_module_debug_logging:
         description: 'Enable debug logs'
         required: false
@@ -66,6 +71,16 @@ on:
         required: false
         default: '16Gi'
         type: string
+      executor_memory_limit:
+        description: 'Memory limit for the executor pod (e.g. 16Gi, 62Gi)'
+        required: false
+        default: '16Gi'
+        type: string
+      custom_image:
+        description: 'Custom runtime container image (e.g. ghcr.io/spiceai/spiceai-dev:spicebench-sf10). Overrides the default channel image. Requires the internal update channel on the target SCP environment.'
+        required: false
+        default: ''
+        type: string
 jobs:
   run-spicebench:
     name: Run spicebench
@@ -134,7 +149,7 @@ jobs:
           SCENARIO: ${{ github.event.inputs.scenario || 'tpch' }}
           SYSTEM_UNDER_TEST: ${{ github.event.inputs.system_under_test || 'spice_cloud' }}
           SYSTEM_ADAPTER: ${{ github.event.inputs.system_under_test || 'spice_cloud' }}
-          NUM_QUERY_CLIENTS: '8'
+          NUM_QUERY_CLIENTS: ${{ github.event.inputs.num_query_clients || '2' }}
           ETL_BUCKET: 'spicebench'
           ETL_PREFIX: ${{ github.event.inputs.etl_type == 'changes' && 'data-gen-mutable' || 'data-gen' }}
           SCALE_FACTOR: ${{ github.event.inputs.scale_factor || '1' }}
@@ -155,6 +170,8 @@ jobs:
           DISABLE_TEARDOWN: ${{ github.event.inputs.disable_teardown || 'false' }}
           ENABLE_PVC: ${{ github.event.inputs.enable_pvc || 'false' }}
           SPIDAPTER_APP_MEMORY_LIMIT: ${{ github.event.inputs.app_memory_limit || '16Gi' }}
+          SPIDAPTER_EXECUTOR_MEMORY_LIMIT: ${{ github.event.inputs.executor_memory_limit || '16Gi' }}
+          CUSTOM_IMAGE: ${{ github.event.inputs.custom_image || '' }}
         run: |
           set -euo pipefail
           if [ "${ENABLE_MODULE_DEBUG_LOGGING}" = "true" ]; then
@@ -201,13 +218,43 @@ jobs:
           ADAPTER_DOCKER_OPTS="run -i -e SPIDAPTER_EXECUTOR_REPLICAS=4 -e SPICEAI_API_KEY -e SPICE_CLOUD_API_URL -e AWS_ACCESS_KEY_ID=${S3_AWS_ACCESS_KEY_ID} -e AWS_SECRET_ACCESS_KEY=${S3_AWS_SECRET_ACCESS_KEY} -e SPIDAPTER_ICEBERG_REGION -e SPIDAPTER_ICEBERG_CATALOG_FROM -e SCHEDULER_STATE_LOCATION"
 
           ADAPTER_DOCKER_OPTS="${ADAPTER_DOCKER_OPTS} -e SPIDAPTER_APP_MEMORY_LIMIT=${SPIDAPTER_APP_MEMORY_LIMIT}"
+          ADAPTER_DOCKER_OPTS="${ADAPTER_DOCKER_OPTS} -e SPIDAPTER_EXECUTOR_MEMORY_LIMIT=${SPIDAPTER_EXECUTOR_MEMORY_LIMIT}"
+
+          # Parse custom image into registry, name, and tag for spidapter.
+          # Expected format: registry/image:tag (e.g. ghcr.io/spiceai/spiceai-dev:spicebench-sf10)
+          # Parses as: registry=ghcr.io, image=spiceai/spiceai-dev, tag=spicebench-sf10
+          if [ -n "${CUSTOM_IMAGE}" ]; then
+            IMAGE_WITH_TAG="${CUSTOM_IMAGE}"
+            if [[ "${IMAGE_WITH_TAG}" == *":"* ]]; then
+              SPIDAPTER_IMAGE_TAG="${IMAGE_WITH_TAG##*:}"
+              IMAGE_WITHOUT_TAG="${IMAGE_WITH_TAG%:*}"
+            else
+              SPIDAPTER_IMAGE_TAG=""
+              IMAGE_WITHOUT_TAG="${IMAGE_WITH_TAG}"
+            fi
+            # Registry is the first path component (hostname), image is the rest
+            SPIDAPTER_IMAGE_REGISTRY="${IMAGE_WITHOUT_TAG%%/*}"
+            SPIDAPTER_IMAGE_NAME="${IMAGE_WITHOUT_TAG#*/}"
+
+            echo "Custom image: registry=${SPIDAPTER_IMAGE_REGISTRY}, name=${SPIDAPTER_IMAGE_NAME}, tag=${SPIDAPTER_IMAGE_TAG}"
+            ADAPTER_DOCKER_OPTS="${ADAPTER_DOCKER_OPTS} -e SPIDAPTER_IMAGE_REGISTRY=${SPIDAPTER_IMAGE_REGISTRY} -e SPIDAPTER_IMAGE_NAME=${SPIDAPTER_IMAGE_NAME}"
+            if [ -n "${SPIDAPTER_IMAGE_TAG}" ]; then
+              ADAPTER_DOCKER_OPTS="${ADAPTER_DOCKER_OPTS} -e SPIDAPTER_IMAGE_TAG=${SPIDAPTER_IMAGE_TAG}"
+            fi
+          fi
 
           if [ "${ENABLE_PVC}" = "true" ]; then
             echo "PVC enabled: app=3GB, executor=2GB"
             ADAPTER_DOCKER_OPTS="${ADAPTER_DOCKER_OPTS} -e SPIDAPTER_APP_STORAGE_SIZE_GB=3 -e SPIDAPTER_EXECUTOR_STORAGE_SIZE_GB=2 -e SPIDAPTER_CAYENNE_DATA_DIR=/data/data -e SPIDAPTER_CAYENNE_METADATA_DIR=/data/metadata"
           fi
 
-          ADAPTER_ARGS="${ADAPTER_DOCKER_OPTS} ghcr.io/spiceai/spidapter:${{ github.event.inputs.spidapter_version || 'latest' }} stdio --verbose --channel nightly"
+          # Use internal channel when a custom image is specified, otherwise nightly.
+          SPIDAPTER_CHANNEL="nightly"
+          if [ -n "${CUSTOM_IMAGE}" ]; then
+            SPIDAPTER_CHANNEL="internal"
+          fi
+
+          ADAPTER_ARGS="${ADAPTER_DOCKER_OPTS} ghcr.io/spiceai/spidapter:${{ github.event.inputs.spidapter_version || 'latest' }} stdio --verbose --channel ${SPIDAPTER_CHANNEL}"
           ADAPTER_ENVS=""
 
           NO_TEARDOWN_ARG=""