Skip to content

Commit 149a968

Browse files
phillipleblancphillipleblanc
authored
feat(testoperator): run cluster benchmarks via system adapters (spiceai#10814)
* feat(testoperator): drive SUT acquisition through system-adapter JSON-RPC Adds the --system-adapter-* flag family + a new cluster-bench workflow that runs testoperator against spidapter-provisioned Spice Cloud clusters. * fix(runtime): backtick DataFusion in cayenne join test doc comment for clippy
1 parent e0904fa commit 149a968

16 files changed

Lines changed: 1042 additions & 94 deletions

File tree

‎.github/workflows/testoperator_run_cluster_bench.yml‎

Lines changed: 267 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,267 @@
1+
name: cluster benchmark e2e tests
2+
run-name: cluster-bench - ${{ github.event.inputs.query_set }} - ${{ github.event.inputs.spicepod_path }}
3+
4+
# Cluster-mode counterpart to `testoperator_run_bench.yml`. Instead of spawning a
5+
# local `spiced` process, testoperator drives the benchmark against a SUT
6+
# acquired via the system-adapter JSON-RPC protocol — here, the `spidapter`
7+
# docker image deploying a Spice Cloud clustered app. Mirrors the
8+
# `run_spicebench.yml` plumbing in the spicebench repo.
9+
10+
on:
11+
workflow_dispatch:
12+
inputs:
13+
spicepod_path:
14+
description: 'The spicepod file to test with (relative to test/spicepods/{query_set}/sf{scale_factor}/)'
15+
required: true
16+
type: string
17+
query_set:
18+
description: 'Query set'
19+
required: true
20+
default: 'tpch'
21+
type: choice
22+
options:
23+
- 'tpch'
24+
- 'tpch[parameterized]'
25+
- 'tpcds'
26+
- 'clickbench'
27+
- 'scenario'
28+
scenario_query_file:
29+
description: 'Path to scenario query file (required when query_set is scenario)'
30+
required: false
31+
type: string
32+
default: ''
33+
query_transport:
34+
description: 'Which distributed path inside the cluster to exercise'
35+
required: true
36+
default: 'flightsql'
37+
type: choice
38+
options:
39+
- 'flightsql' # Distributed accelerations (Flight SQL gRPC)
40+
- 'v1-queries' # Ballista distributed query (HTTP /v1/queries) — sets --distributed
41+
executor_replicas:
42+
description: 'Number of spidapter executor replicas'
43+
required: true
44+
default: '4'
45+
type: string
46+
channel:
47+
description: 'Spice runtime image channel passed to spidapter'
48+
required: true
49+
default: 'nightly'
50+
type: choice
51+
options:
52+
- 'nightly'
53+
- 'preview'
54+
- 'stable'
55+
- 'internal'
56+
scale_factor:
57+
description: 'Scale factor for the benchmark'
58+
required: false
59+
type: string
60+
default: '1'
61+
validate_results:
62+
description: 'Validate query results against expected (where supported)'
63+
required: false
64+
type: boolean
65+
default: false
66+
update_snapshots:
67+
description: |
68+
How to handle insta snapshot mismatches. `default` decides based on
69+
the branch (release/* → no, anything else → always). `always` creates
70+
new baselines on mismatch (use this on a brand-new spicepod that
71+
has no snapshots yet). `no` fails on any drift, matching production
72+
benchmark behavior. Mirrors `testoperator_run_bench.yml`.
73+
required: false
74+
default: 'default'
75+
type: choice
76+
options:
77+
- 'default'
78+
- 'always'
79+
- 'no'
80+
ready_wait:
81+
description: 'Seconds to wait for the cluster to become ready'
82+
required: true
83+
default: '600'
84+
type: string
85+
runner_type:
86+
description: 'Runner type'
87+
required: true
88+
default: 'spiceai-dev-runners'
89+
type: choice
90+
options:
91+
- 'spiceai-dev-runners'
92+
- 'spiceai-dev-large-runners'
93+
94+
permissions:
95+
contents: read
96+
packages: read
97+
98+
# Centralized defaults so push-triggered runs (where github.event.inputs.* is
99+
# empty) and workflow_dispatch runs share the same fallback chain. Configured
100+
# for a Ballista distributed-query smoke test against the public-S3 TPC-H
101+
# spicepod by default.
102+
env:
103+
CB_SPICEPOD_PATH: ${{ github.event.inputs.spicepod_path || 'federated/s3-public[parquet].yaml' }}
104+
CB_QUERY_SET: ${{ github.event.inputs.query_set || 'tpch' }}
105+
CB_SCENARIO_QUERY_FILE: ${{ github.event.inputs.scenario_query_file || '' }}
106+
CB_QUERY_TRANSPORT: ${{ github.event.inputs.query_transport || 'v1-queries' }}
107+
CB_EXECUTOR_REPLICAS: ${{ github.event.inputs.executor_replicas || '3' }}
108+
CB_CHANNEL: ${{ github.event.inputs.channel || 'nightly' }}
109+
CB_SCALE_FACTOR: ${{ github.event.inputs.scale_factor || '1' }}
110+
CB_VALIDATE_RESULTS: ${{ github.event.inputs.validate_results || 'false' }}
111+
CB_UPDATE_SNAPSHOTS: ${{ github.event.inputs.update_snapshots || 'default' }}
112+
CB_READY_WAIT: ${{ github.event.inputs.ready_wait || '600' }}
113+
CB_RUNNER_TYPE: ${{ github.event.inputs.runner_type || 'spiceai-dev-runners' }}
114+
115+
jobs:
116+
run-cluster-bench:
117+
name: Run cluster benchmark
118+
runs-on: ${{ github.event.inputs.runner_type || 'spiceai-dev-runners' }}
119+
timeout-minutes: 600
120+
steps:
121+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
122+
123+
- name: Set up Rust toolchain
124+
uses: ./.github/actions/setup-rust
125+
126+
- name: Determine snapshot update mode
127+
id: determine_update_snapshots
128+
run: |
129+
# `default` → branch-based: release/* never updates, anything else
130+
# always updates. Mirrors testoperator_run_bench.yml so cluster-bench
131+
# snapshot policy is consistent with the single-node bench.
132+
update_mode="${CB_UPDATE_SNAPSHOTS}"
133+
if [ "${update_mode}" = "default" ] || [ -z "${update_mode}" ]; then
134+
branch_name="${{ github.ref_name }}"
135+
if [[ "${branch_name}" == "release/"* ]]; then
136+
update_mode="no"
137+
else
138+
update_mode="always"
139+
fi
140+
fi
141+
142+
echo "Using update_snapshots=${update_mode}" >&2
143+
echo "update_snapshots=${update_mode}" >> "${GITHUB_OUTPUT}"
144+
145+
- name: Set spicepod path
146+
id: set_spicepod_path
147+
run: |
148+
query_set="${CB_QUERY_SET}"
149+
query_set="${query_set%%\[*\]*}"
150+
SPICEPOD_PATH="./test/spicepods/${query_set}/sf${CB_SCALE_FACTOR}/${CB_SPICEPOD_PATH}"
151+
echo "SPICEPOD_PATH=${SPICEPOD_PATH}" >> $GITHUB_OUTPUT
152+
153+
- name: Validate spicepod file exists
154+
run: |
155+
if [ ! -f "${{ steps.set_spicepod_path.outputs.SPICEPOD_PATH }}" ]; then
156+
echo "Error: Spicepod file not found at ${{ steps.set_spicepod_path.outputs.SPICEPOD_PATH }}"
157+
exit 1
158+
fi
159+
echo "Spicepod file found at ${{ steps.set_spicepod_path.outputs.SPICEPOD_PATH }}"
160+
161+
- name: Install MinIO
162+
uses: ./.github/actions/setup-minio
163+
with:
164+
minio_endpoint: ${{ secrets.TEST_MINIO_ENDPOINT }}
165+
minio_access_key: ${{ secrets.TEST_MINIO_ACCESS_KEY }}
166+
minio_secret_key: ${{ secrets.TEST_MINIO_SECRET_KEY }}
167+
168+
- name: Build spicepod validator
169+
uses: ./.github/actions/build-spicepod-validator
170+
with:
171+
minio_endpoint: ${{ secrets.TEST_MINIO_ENDPOINT }}
172+
minio_access_key: ${{ secrets.TEST_MINIO_ACCESS_KEY }}
173+
minio_secret_key: ${{ secrets.TEST_MINIO_SECRET_KEY }}
174+
175+
- name: Validate spicepod
176+
run: |
177+
spicepod-validator "${{ steps.set_spicepod_path.outputs.SPICEPOD_PATH }}"
178+
179+
- name: Build Testoperator
180+
uses: ./.github/actions/build-testoperator
181+
with:
182+
minio_endpoint: ${{ secrets.TEST_MINIO_ENDPOINT }}
183+
minio_access_key: ${{ secrets.TEST_MINIO_ACCESS_KEY }}
184+
minio_secret_key: ${{ secrets.TEST_MINIO_SECRET_KEY }}
185+
186+
- name: Setup Testoperator
187+
uses: ./.github/actions/setup-testoperator-data
188+
with:
189+
query_set: ${{ env.CB_QUERY_SET }}
190+
191+
- name: Build Spidapter
192+
uses: ./.github/actions/build-spidapter
193+
with:
194+
minio_endpoint: ${{ secrets.TEST_MINIO_ENDPOINT }}
195+
minio_access_key: ${{ secrets.TEST_MINIO_ACCESS_KEY }}
196+
minio_secret_key: ${{ secrets.TEST_MINIO_SECRET_KEY }}
197+
198+
- name: Management login (prod)
199+
uses: ./.github/actions/management-login
200+
with:
201+
token-url: https://spice.ai/api/oauth/token
202+
client-id: ${{ secrets.SPICE_MANAGEMENT_CLIENT_ID_PROD }}
203+
client-secret: ${{ secrets.SPICE_MANAGEMENT_CLIENT_SECRET_PROD }}
204+
# Exports SPICEAI_API_KEY for subsequent steps. Same prod creds
205+
# spicebench's run_spicebench.yml uses.
206+
207+
- name: Run the cluster benchmark - ${{ env.CB_SPICEPOD_PATH }}
208+
env:
209+
SPICEAI_API_KEY: ${{ env.SPICEAI_API_KEY }}
210+
SPICE_CLOUD_API_URL: https://api.spice.ai
211+
S3_ENDPOINT: ${{ secrets.TEST_MINIO_ENDPOINT }}
212+
S3_KEY: ${{ secrets.TEST_MINIO_ACCESS_KEY }}
213+
S3_SECRET: ${{ secrets.TEST_MINIO_SECRET_KEY }}
214+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ICEBERG_ACCESS_KEY_ID }}
215+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ICEBERG_SECRET_ACCESS_KEY }}
216+
SCHEDULER_STATE_LOCATION: 's3://spiceai-testing-cluster-state/testoperator-cluster-bench-${{ github.run_id }}/'
217+
SPICEAI_BENCHMARK_METRICS_KEY: ${{ secrets.SPICEAI_BENCHMARK_METRICS_KEY }}
218+
EXECUTOR_REPLICAS: ${{ env.CB_EXECUTOR_REPLICAS }}
219+
CHANNEL: ${{ env.CB_CHANNEL }}
220+
QUERY_SET: ${{ env.CB_QUERY_SET }}
221+
QUERY_TRANSPORT: ${{ env.CB_QUERY_TRANSPORT }}
222+
SCALE_FACTOR: ${{ env.CB_SCALE_FACTOR }}
223+
VALIDATE_RESULTS: ${{ env.CB_VALIDATE_RESULTS }}
224+
READY_WAIT: ${{ env.CB_READY_WAIT }}
225+
INSTA_UPDATE: ${{ steps.determine_update_snapshots.outputs.update_snapshots }}
226+
SPICEPOD_PATH: ${{ steps.set_spicepod_path.outputs.SPICEPOD_PATH }}
227+
SCENARIO_QUERY_FILE: ${{ env.CB_SCENARIO_QUERY_FILE }}
228+
run: |
229+
set -euo pipefail
230+
231+
# Translate the abstract transport name into the concrete testoperator
232+
# flag. `--distributed` selects the existing DistributedExecutor that
233+
# POSTs to /v1/queries; the default (no flag) keeps the Flight SQL
234+
# path that exercises distributed accelerations.
235+
TRANSPORT_FLAG=""
236+
if [ "${QUERY_TRANSPORT}" = "v1-queries" ]; then
237+
TRANSPORT_FLAG="--distributed"
238+
fi
239+
240+
# Optional scenario query file (only used when --query-set scenario).
241+
SCENARIO_ARG=""
242+
if [ -n "${SCENARIO_QUERY_FILE}" ]; then
243+
SCENARIO_ARG="--scenario-query-file ./test/scenario/${SCENARIO_QUERY_FILE}"
244+
fi
245+
246+
# Run the freshly-built spidapter binary as the JSON-RPC adapter.
247+
# spidapter reads its config from SPIDAPTER_*/SPICEAI_*/AWS_*/SCHEDULER_*
248+
# env vars, which testoperator's child process inherits from this step's
249+
# env block.
250+
export SPIDAPTER_EXECUTOR_REPLICAS="${EXECUTOR_REPLICAS}"
251+
252+
rm -rf .spice/data
253+
INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" testoperator run bench \
254+
-p "${SPICEPOD_PATH}" \
255+
--query-set "${QUERY_SET}" \
256+
--scale-factor "${SCALE_FACTOR}" \
257+
--ready-wait "${READY_WAIT}" \
258+
--disable-progress-bars \
259+
--validate="${VALIDATE_RESULTS}" \
260+
--metrics \
261+
${TRANSPORT_FLAG} \
262+
${SCENARIO_ARG} \
263+
--system-adapter-stdio-cmd spidapter \
264+
--system-adapter-stdio-args "stdio --verbose --channel ${CHANNEL}" \
265+
--system-adapter-param channel="${CHANNEL}" \
266+
--system-adapter-param executor_replicas="${EXECUTOR_REPLICAS}" \
267+
--system-adapter-param query_transport="${QUERY_TRANSPORT}"

‎Cargo.lock‎

Lines changed: 5 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -294,7 +294,7 @@ sqlparser = "0.59.0"
294294
ssh2 = { version = "0.9.5" }
295295
suppaftp = { version = "6.3.0", features = ["async"] }
296296
sysinfo = "0.38.4"
297-
system-adapter-protocol = { git = "https://github.com/spiceai/spicebench.git", rev = "6327983bc1a90123e0c754b79781b931c969831e", features = ["server"] }
297+
system-adapter-protocol = { git = "https://github.com/spiceai/spicebench.git", rev = "0152edc7ba5a6c0fd471c0ef6ef074cce8b8f830", features = ["client", "server"] }
298298
tantivy = "0.26.0"
299299
tempfile = "3"
300300
tiberius = { version = "0.12.3", default-features = false, features = [

0 commit comments

Comments
 (0)