Reintroduce BENCH_RESTARTABLE; skip stop/start for no-daemon systems

alexey-milovidov · claude · alexey-milovidov · commit c7b93a7ed191 · 2026-05-12T19:09:15.000+02:00
Embedded-CLI systems (clickhouse-local variants, duckdb, datafusion,
sqlite, hyper, chdb, spark variants, etc.) have no-op start/stop and
their ./check always succeeds. The unified driver still ran the cold
cycle stop -&gt; wait_stopped -&gt; drop_caches -&gt; start -&gt; check on every
query, and bench_wait_stopped would burn its full 60s timeout each
time because ./check never starts failing. Across the query sweep
that's ~25-30 minutes of pure idle wait per benchmark run.

Reintroduce BENCH_RESTARTABLE (default yes) as a flag independent of
BENCH_DURABLE. When no, bench_run_query and bench_concurrent_qps skip
the stop/wait/start/check dance and only drop_caches between queries.
Set BENCH_RESTARTABLE=no on the 34 shims whose start/stop are pure
no-ops.

Co-Authored-By: Claude Opus 4.7 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/chdb-parquet-partitioned/benchmark.sh b/chdb-parquet-partitioned/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/chdb/benchmark.sh b/chdb/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-csv"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/chyt/benchmark.sh b/chyt/benchmark.sh
@@ -5,4 +5,5 @@ export YT_USE_HOSTS=0
 export CHYT_ALIAS="${CHYT_ALIAS:-*ch_public}"
 export BENCH_DOWNLOAD_SCRIPT=""
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/clickhouse-datalake-partitioned/benchmark.sh b/clickhouse-datalake-partitioned/benchmark.sh
@@ -3,4 +3,5 @@
 # Data is read directly from S3, no local download.
 export BENCH_DOWNLOAD_SCRIPT=""
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/clickhouse-datalake/benchmark.sh b/clickhouse-datalake/benchmark.sh
@@ -3,4 +3,5 @@
 # Data is read directly from S3, no local download.
 export BENCH_DOWNLOAD_SCRIPT=""
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/clickhouse-parquet-partitioned/benchmark.sh b/clickhouse-parquet-partitioned/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/clickhouse-parquet/benchmark.sh b/clickhouse-parquet/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/datafusion-partitioned/benchmark.sh b/datafusion-partitioned/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/datafusion-vortex-partitioned/benchmark.sh b/datafusion-vortex-partitioned/benchmark.sh
@@ -3,4 +3,5 @@
 # query_bench (the vortex driver) handles its own dataset download/conversion.
 export BENCH_DOWNLOAD_SCRIPT=""
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/datafusion-vortex/benchmark.sh b/datafusion-vortex/benchmark.sh
@@ -3,4 +3,5 @@
 # clickbench (the vortex driver) handles its own dataset download/conversion.
 export BENCH_DOWNLOAD_SCRIPT=""
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/datafusion/benchmark.sh b/datafusion/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/drill/benchmark.sh b/drill/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/duckdb-datalake-partitioned/benchmark.sh b/duckdb-datalake-partitioned/benchmark.sh
@@ -3,4 +3,5 @@
 # Data is read directly from S3, no local download.
 export BENCH_DOWNLOAD_SCRIPT=""
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/duckdb-datalake/benchmark.sh b/duckdb-datalake/benchmark.sh
@@ -3,4 +3,5 @@
 # Data is read directly from S3, no local download.
 export BENCH_DOWNLOAD_SCRIPT=""
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/duckdb-parquet-partitioned/benchmark.sh b/duckdb-parquet-partitioned/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/duckdb-parquet/benchmark.sh b/duckdb-parquet/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/duckdb-vortex-partitioned/benchmark.sh b/duckdb-vortex-partitioned/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/duckdb-vortex/benchmark.sh b/duckdb-vortex/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/duckdb/benchmark.sh b/duckdb/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/glaredb-partitioned/benchmark.sh b/glaredb-partitioned/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/glaredb/benchmark.sh b/glaredb/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/hyper-parquet/benchmark.sh b/hyper-parquet/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/hyper/benchmark.sh b/hyper/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-csv"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/lib/benchmark-common.sh b/lib/benchmark-common.sh
@@ -13,6 +13,23 @@
 #                          from a remote source (S3 datalake, remote services).
 #
 # Optional env:
+#   BENCH_RESTARTABLE      "yes" (default) or "no". Tells the driver whether
+#                          stop/start is meaningful for this system.
+#                            yes — system has a daemon (or in-process server
+#                                  wrapper) whose lifecycle matters. The
+#                                  cold cycle is stop -> wait_stopped ->
+#                                  drop_caches -> start -> check.
+#                            no  — system has no daemon: ./start, ./stop,
+#                                  and ./check are no-ops (embedded CLIs
+#                                  like clickhouse-local, duckdb,
+#                                  datafusion, sqlite, hyper, chdb,
+#                                  spark variants).
+#                                  Restarting would do nothing, and worse,
+#                                  bench_wait_stopped would spin until its
+#                                  60s timeout on every cold cycle because
+#                                  ./check keeps succeeding. We skip the
+#                                  stop/start/check dance entirely and
+#                                  just drop_caches between queries.
 #   BENCH_DURABLE          "yes" (default) or "no". Tells the driver whether
 #                          the system's data survives a stop+start.
 #                            yes — data is on disk (daemons like clickhouse,
@@ -57,11 +74,7 @@ export HOME="${HOME:-/root}"
 
 # BENCH_DOWNLOAD_SCRIPT must be set (possibly to empty for "no download").
 : "${BENCH_DOWNLOAD_SCRIPT?BENCH_DOWNLOAD_SCRIPT is required (set empty to skip)}"
-# Back-compat: accept the old BENCH_RESTARTABLE name as an alias for one
-# release cycle. Operators with stale env files / scripts still work.
-if [ -n "${BENCH_RESTARTABLE:-}" ] && [ -z "${BENCH_DURABLE:-}" ]; then
-    BENCH_DURABLE="$BENCH_RESTARTABLE"
-fi
+: "${BENCH_RESTARTABLE:=yes}"
 : "${BENCH_DURABLE:=yes}"
 : "${BENCH_TRIES:=3}"
 : "${BENCH_QUERIES_FILE:=queries.sql}"
@@ -208,11 +221,20 @@ bench_run_query() {
     # Waiting for ./check to fail before flushing makes the cold run
     # actually cold even when ./stop returns before the process is
     # fully gone.
-    ./stop >/dev/null 2>&1 || true
-    bench_wait_stopped
-    bench_flush_caches
-    ./start >/dev/null 2>&1 || true
-    bench_check_loop
+    #
+    # BENCH_RESTARTABLE=no skips the stop/wait/start/check dance: those
+    # scripts are no-ops for embedded-CLI systems and bench_wait_stopped
+    # would otherwise burn its full 60s timeout on every query (./check
+    # never starts failing because nothing was actually running).
+    if [ "$BENCH_RESTARTABLE" = "yes" ]; then
+        ./stop >/dev/null 2>&1 || true
+        bench_wait_stopped
+        bench_flush_caches
+        ./start >/dev/null 2>&1 || true
+        bench_check_loop
+    else
+        bench_flush_caches
+    fi
 
     if [ "$BENCH_DURABLE" != "yes" ]; then
         # In-process server: data lived in process memory and was wiped
@@ -316,12 +338,17 @@ bench_concurrent_qps() {
         return 0
     fi
 
-    # Fresh restart before the throughput window starts.
-    ./stop >/dev/null 2>&1 || true
-    bench_wait_stopped
-    bench_flush_caches
-    ./start >/dev/null 2>&1 || true
-    bench_check_loop
+    # Fresh restart before the throughput window starts. Mirrors the
+    # BENCH_RESTARTABLE handling in bench_run_query.
+    if [ "$BENCH_RESTARTABLE" = "yes" ]; then
+        ./stop >/dev/null 2>&1 || true
+        bench_wait_stopped
+        bench_flush_caches
+        ./start >/dev/null 2>&1 || true
+        bench_check_loop
+    else
+        bench_flush_caches
+    fi
     if [ "$BENCH_DURABLE" != "yes" ]; then
         ./load >/dev/null 2>&1 || true
     fi
diff --git a/octosql/benchmark.sh b/octosql/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/opteryx/benchmark.sh b/opteryx/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/sail-partitioned/benchmark.sh b/sail-partitioned/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/sail/benchmark.sh b/sail/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/spark-auron/benchmark.sh b/spark-auron/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/spark-comet/benchmark.sh b/spark-comet/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/spark-gluten/benchmark.sh b/spark-gluten/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/spark-velox/benchmark.sh b/spark-velox/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/spark/benchmark.sh b/spark/benchmark.sh
@@ -4,4 +4,5 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
 # Spark runs in-process per query — restart between queries is meaningless
 # (and would re-download nothing). Skip restart.
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/sqlite/benchmark.sh b/sqlite/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-csv"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
diff --git a/turso/benchmark.sh b/turso/benchmark.sh
@@ -2,4 +2,5 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-csv"
 export BENCH_DURABLE=yes
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh
