sirius-db · aocsa · Sep 3, 2026 · Sep 5, 2026 · Sep 5, 2026 · Sep 5, 2026
diff --git a/bench/common/gen-tpch.sh b/bench/common/gen-tpch.sh
@@ -0,0 +1,147 @@
+#!/usr/bin/env bash
+# Generate a TPC-H parquet dataset at any scale factor.
+#
+# Defaults to f64 monetary columns. decimal128 activates Sirius's decimal-lowering drift,
+# which scores ~11/22 against a DuckDB oracle instead of ~21/22, so f64 is what the
+# benchmark baselines use. Pass --decimal only when you are deliberately testing that path.
+set -euo pipefail
+
+SCALE=""
+DECIMAL_TYPE=f64
+OUT=""
+ROOT=${TPCH_ROOT:-/opt/dlami/nvme/tpch}
+GEN=${TPCHGEN:-$ROOT/tpchgen-rs/target/release/tpchgen-cli}
+PYTHON=${TPCH_PYTHON:-$ROOT/venv/bin/python}
+TARGET_PART_GB=${TARGET_PART_GB:-4.3}
+FORCE=0
+VERIFY=1
+
+usage() {
+    cat <<USAGE
+usage: gen-tpch.sh -s <scale> [options]
+
+  -s, --scale <N>       scale factor (required)
+  -o, --output <dir>    output directory
+                        (default: \$TPCH_ROOT/tpch_parquet_sf<N>[_f64])
+      --decimal         decimal128 monetary columns (default: f64)
+      --force           overwrite an existing output directory
+      --no-verify       skip the row-count check
+  -h, --help
+
+env: TPCH_ROOT (default /opt/dlami/nvme/tpch), TPCHGEN, TPCH_PYTHON, TARGET_PART_GB
+USAGE
+}
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        -s|--scale)   SCALE=${2:?}; shift 2 ;;
+        -o|--output)  OUT=${2:?}; shift 2 ;;
+        --decimal)    DECIMAL_TYPE=decimal128; shift ;;
+        --force)      FORCE=1; shift ;;
+        --no-verify)  VERIFY=0; shift ;;
+        -h|--help)    usage; exit 0 ;;
+        *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+    esac
+done
+
+die() { echo "FATAL: $*" >&2; exit 1; }
+
+[ -n "$SCALE" ] || { usage >&2; die "-s/--scale is required"; }
+[[ "$SCALE" =~ ^[0-9]+$ ]] && [ "$SCALE" -gt 0 ] || die "scale must be a positive integer, got '$SCALE'"
+[ -x "$GEN" ] || die "tpchgen-cli not executable at $GEN (set TPCHGEN)"
+
+if [ -z "$OUT" ]; then
+    OUT=$ROOT/tpch_parquet_sf$SCALE
+    [ "$DECIMAL_TYPE" = f64 ] && OUT=${OUT}_f64
+fi
+
+if [ -e "$OUT" ] && [ -n "$(ls -A "$OUT" 2>/dev/null)" ]; then
+    [ "$FORCE" = 1 ] || die "$OUT exists and is not empty; pass --force to overwrite"
+    rm -rf "$OUT"
+fi
+mkdir -p "$OUT"
+
+# Approximate compressed GB per unit of scale factor, measured from the SF300 and SF500 sets.
+# Part counts derived from these reproduce the existing SF100/SF300/SF500 layouts.
+gb_per_sf() {
+    case "$1" in
+        lineitem) echo 0.258 ;;
+        orders)   echo 0.075 ;;
+        partsupp) echo 0.047 ;;
+        customer) echo 0.0138 ;;
+        part)     echo 0.0068 ;;
+        supplier) echo 0.0009 ;;
+        *)        echo 0 ;;
+    esac
+}
+
+parts_for() {
+    local table=$1
+    case "$table" in nation|region) echo 1; return ;; esac
+    "$PYTHON" - "$(gb_per_sf "$table")" "$SCALE" "$TARGET_PART_GB" <<'PY'
+import sys
+per_sf, scale, target = float(sys.argv[1]), float(sys.argv[2]), float(sys.argv[3])
+print(max(1, round(per_sf * scale / target)))
+PY
+}
+
+TABLES="region nation supplier part customer partsupp orders lineitem"
+
+echo "scale=$SCALE  decimal=$DECIMAL_TYPE  out=$OUT"
+for t in $TABLES; do
+    n=$(parts_for "$t")
+    echo "[$(date +%T)] $t parts=$n"
+    "$GEN" -s "$SCALE" -T "$t" -f parquet --parts "$n" \
+           --decimal-column-type "$DECIMAL_TYPE" -o "$OUT"
+done
+
+cat > "$OUT/gen-info.json" <<JSON
+{
+  "scale_factor": $SCALE,
+  "decimal_column_type": "$DECIMAL_TYPE",
+  "target_part_gb": $TARGET_PART_GB,
+  "generated_utc": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
+  "generator": "$($GEN --version 2>/dev/null | head -1)"
+}
+JSON
+
+echo "[$(date +%T)] generated $(find "$OUT" -name '*.parquet' | wc -l) files, $(du -sh "$OUT" | cut -f1)"
+
+[ "$VERIFY" = 1 ] || { echo "verification skipped"; exit 0; }
+[ -x "$PYTHON" ] || die "no python with duckdb at $PYTHON (set TPCH_PYTHON or pass --no-verify)"
+
+# Every table but lineitem has an exact per-SF row count. lineitem's varies slightly with the
+# generator's data distribution, so it is checked against a tolerance instead.
+"$PYTHON" - "$OUT" "$SCALE" "$DECIMAL_TYPE" <<'PY'
+import sys, duckdb
+out, scale, dtype = sys.argv[1], int(sys.argv[2]), sys.argv[3]
+exact = {"orders":1500000,"customer":150000,"part":200000,"partsupp":800000,"supplier":10000}
+con, bad = duckdb.connect(), []
+for t, per in sorted(exact.items()):
+    n = con.sql(f"select count(*) from read_parquet('{out}/{t}/*.parquet')").fetchone()[0]
+    e = per*scale
+    ok = n == e
+    bad += [] if ok else [t]
+    print(f"{t:10s} {n:>15,} expected {e:>15,} {'OK' if ok else 'MISMATCH'}")
+for t, e in (("nation",25),("region",5)):
+    n = con.sql(f"select count(*) from read_parquet('{out}/{t}/*.parquet')").fetchone()[0]
+    bad += [] if n == e else [t]
+    print(f"{t:10s} {n:>15,} expected {e:>15,} {'OK' if n==e else 'MISMATCH'}")
+n = con.sql(f"select count(*) from read_parquet('{out}/lineitem/*.parquet')").fetchone()[0]
+e = 6001215*scale
+ok = abs(n-e)/e < 0.001
+bad += [] if ok else ["lineitem"]
+print(f"{'lineitem':10s} {n:>15,} ~expected {e:>15,} ({(n-e)/e*100:+.3f}%) {'OK' if ok else 'MISMATCH'}")
+
+want = "DOUBLE" if dtype == "f64" else "DECIMAL"
+got = dict(con.sql(
+    f"select column_name, column_type from (describe select * from "
+    f"read_parquet('{out}/lineitem/*.parquet')) "
+    f"where column_name in ('l_quantity','l_extendedprice','l_discount','l_tax')").fetchall())
+wrong = [c for c, t in got.items() if want not in t]
+print(f"monetary columns: {sorted(set(got.values()))} expected {want}")
+if wrong: bad.append("column-types")
+if bad:
+    print("FAILED: " + ", ".join(sorted(set(bad)))); sys.exit(1)
+print("all row counts and column types OK")
+PY
@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+# Bring up 1 FE + N Sirius GPU compute nodes, one CN per GPU, cross-node exchange over nixl.
+#
+# The `cluster2` pixi task generalized to a loop. Two CNs could offset their ports by +2; at
+# eight the heartbeat range would collide with the thrift range, so each CN instead gets a
+# contiguous 10-port block based at $PORT_BASE -- clear of the FE's ports (8030/9010/9020/9030)
+# and of the CN defaults (9050/9060/8040/8060/9070).
+#
+# Two identities must stay unique across CNs: the FE keys a node by
+# (advertise_host, heartbeat_port), and the nixl agent is named {advertise_host}:{brpc_port}.
+#
+# Usage:  ./benchmarks/cluster8.sh
+#         NUM_CNS=4 GPU_MEM=48GiB ./benchmarks/cluster8.sh
+#         SIRIUS_QUERY_WATCHDOG_SECS=60 NUM_CNS=4 ./benchmarks/cluster8.sh
+#
+# Run it in its own terminal or as its own background task -- never chained behind `&` inside
+# another shell command, or the cluster dies with that shell.
+set -euo pipefail
+
+HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+SR_DIR=$(cd "$HERE/.." && pwd)              # experimental/starrocks
+REPO_ROOT=$(cd "$SR_DIR/../.." && pwd)
+TOOLS_DIR=${TOOLS_DIR:-$(cd "$REPO_ROOT/.." && pwd)/tools}
+
+NUM_CNS=${NUM_CNS:-8}
+PORT_BASE=${PORT_BASE:-9100}
+PORT_STRIDE=${PORT_STRIDE:-10}
+# The staging arena sits OUTSIDE --gpu-memory-limit, so a CN really occupies
+# GPU_MEM + STAGING + CUDA context. On an 80GiB A100 that is ~75 of 80 GiB.
+GPU_MEM=${GPU_MEM:-64GiB}
+HOST_MEM=${HOST_MEM:-128GiB}
+STAGING=${STAGING:-8GiB}
+
+CN_BIN=$SR_DIR/target/release/sirius-starrocks-cn
+FE_BIN=$SR_DIR/starrocks/output/fe/bin/start_fe.sh
+
+[ -x "$CN_BIN" ] || { echo "no CN binary at $CN_BIN -- run: pixi run cn-build" >&2; exit 1; }
+[ -x "$FE_BIN" ] || { echo "no packaged FE at $FE_BIN -- run: pixi run fe-check" >&2; exit 1; }
+
+# NIXL_PREFIX / NIXL_PLUGIN_DIR / LD_LIBRARY_PATH (engine .so + pixi env lib + nixl + UCX) /
+# UCX_TLS, all derived from the repo and $TOOLS_DIR locations; fails loudly when nixl is
+# absent rather than continuing misconfigured.
+# shellcheck source=../scripts/cn-env.sh
+source "$SR_DIR/scripts/cn-env.sh"
+
+export SIRIUS_EXCHANGE_STAGING_BYTES=${SIRIUS_EXCHANGE_STAGING_BYTES:-$STAGING}
+
+# Each CN is pinned to its GPU with --gpu-device. An exported CUDA_VISIBLE_DEVICES wins over
+# that flag at the driver level -- every CN on one GPU, a cluster that still answers queries and
+# still records numbers -- which is why the CN refuses to start when the two disagree. Clear it
+# here instead of relying on the operator's shell being clean.
+unset CUDA_VISIBLE_DEVICES
+
+# Engine stall watchdog: seconds of zero scheduling progress before a query is failed loudly
+# instead of wedging the CN (0 = off, the engine default). Exported explicitly so the knob
+# reaches every CN; sweeps that would otherwise record a silent wedge as a timeout set it to 60.
+export SIRIUS_QUERY_WATCHDOG_SECS=${SIRIUS_QUERY_WATCHDOG_SECS:-0}
+
+# GPU_DEVICES="0,0" places CN i on the i-th listed device instead of device i, so two CNs can
+# share one card on a single-GPU box (each must then carve out a GPU_MEM + STAGING slice that
+# fits alongside the other's). Default: one CN per GPU.
+# MIG_DEVICES="MIG-<uuid>,MIG-<uuid>" places CN i on the i-th MIG instance. A MIG instance is only
+# reachable through CUDA_VISIBLE_DEVICES=<uuid>, and the CN rejects a UUID next to --gpu-device, so
+# each CN is launched with the variable exported and without the flag.
+avail=$(nvidia-smi --query-gpu=index --format=csv,noheader | wc -l)
+mig_of_cn=()
+if [ -n "${MIG_DEVICES:-}" ]; then
+    IFS=, read -r -a mig_of_cn <<< "$MIG_DEVICES"
+    [ "${#mig_of_cn[@]}" -ge "$NUM_CNS" ] || {
+        echo "MIG_DEVICES=$MIG_DEVICES lists ${#mig_of_cn[@]} instances for $NUM_CNS CNs" >&2; exit 1; }
+    gpu_of_cn=()
+elif [ -n "${GPU_DEVICES:-}" ]; then
+    IFS=, read -r -a gpu_of_cn <<< "$GPU_DEVICES"
+    [ "${#gpu_of_cn[@]}" -ge "$NUM_CNS" ] || {
+        echo "GPU_DEVICES=$GPU_DEVICES lists ${#gpu_of_cn[@]} devices for $NUM_CNS CNs" >&2; exit 1; }
+else
+    [ "$avail" -ge "$NUM_CNS" ] || {
+        echo "asked for $NUM_CNS CNs but only $avail GPUs are visible" >&2; exit 1; }
+    gpu_of_cn=($(seq 0 $((NUM_CNS - 1))))
+fi
+
+pids=()
+cleanup() {
+    status=$?
+    trap - EXIT INT TERM
+    kill "${pids[@]}" 2>/dev/null || true
+    wait "${pids[@]}" 2>/dev/null || true
+    exit "$status"
+}
+trap cleanup EXIT INT TERM
+
+cd "$SR_DIR"
+"$FE_BIN" --logconsole &
+pids+=("$!")
+
+for i in $(seq 0 $((NUM_CNS - 1))); do
+    base=$((PORT_BASE + i * PORT_STRIDE))
+    if [ "${#mig_of_cn[@]}" -gt 0 ]; then
+        gpu=${mig_of_cn[$i]}; gpu_args=(); export CUDA_VISIBLE_DEVICES=$gpu
+    else
+        gpu=${gpu_of_cn[$i]}; gpu_args=(--gpu-device "$gpu"); unset CUDA_VISIBLE_DEVICES
+    fi
+    "$CN_BIN" \
+        "${gpu_args[@]}" \
+        --heartbeat-port "$base" \
+        --thrift-port    "$((base + 1))" \
+        --brpc-port      "$((base + 2))" \
+        --http-port      "$((base + 3))" \
+        --starlet-port   "$((base + 4))" \
+        --gpu-memory-limit "$GPU_MEM" \
+        --host-memory-limit "$HOST_MEM" \
+        --engine-dir ".cn$i" &
+    pids+=("$!")
+    echo "CN$i gpu=$gpu heartbeat=$base brpc=$((base + 2)) pid=${pids[-1]}"
+done
+
+echo "FE + $NUM_CNS CNs launched; each CN self-registers with the FE on :9030"
+# Any child exiting means the cluster is broken -- fall through to cleanup rather than
+# leaving a half-cluster that the benchmark would silently measure.
+wait -n "${pids[@]}"
@@ -0,0 +1,77 @@
+# TPC-H query deviations from the stock text
+
+Every deviation from the stock TPC-H query text, why it exists, and what it costs.
+
+**These notes live here and NOT as `--` comments inside the `.sql` files.** `bench.sh` runs each
+query as `mysql -e "$Q"`, which collapses newlines — a leading `--` comment therefore swallows the
+entire statement and the query fails with a syntax error. Verified the hard way.
+
+---
+
+## q11 — fraction scaled by SF, `ORDER BY value DESC, ps_partkey` (2026-09-05)
+
+The HAVING fraction is `0.0001 / __TPCH_SF__` (substituted from `$TPCH_SF`, default 1), which is
+the spec's `0.0001 / SF`; the stock constant returns zero rows at SF1000 on every engine.
+
+The spec orders by `value DESC` only. At SF1000 the query returns 936,989 rows and thousands of
+them share a `value`, so two correct engines order the ties differently and a row-order-sensitive
+compare (`tools/compare.py`) reports `VALUES-DIFFER` with a huge `maxreldiff` on nothing but tie
+order (arm W1-1cn: 12,790 mismatched cells, every one a swapped tie). `ps_partkey` is unique per
+output row, so appending it makes the order total. Same text on both engines, same result set.
+
+---
+
+## q08, q09 — `FROM` clause reordered (2026-08-19)
+
+```diff
+     FROM
+         part,
+-        supplier,
+-        lineitem,
++        lineitem,
++        supplier,
+         orders,          -- q08
+         partsupp,        -- q09
+```
+
+Semantically identical: these are inner joins, which commute. **But it is not the stock text, so
+both engines in an A/B comparison must use THIS text or the comparison is invalid.**
+
+### Why
+
+The FE has **no statistics for `FILES()` external scans** — every node in the plan reports
+`cardinality: 1` (see `plans/q08.verbose.txt`). With no cost signal the CBO joins `part` and
+`supplier` first: they are adjacent in the stock `FROM` and share **no predicate** (both keys
+route through `lineitem`). The result is `4:NESTLOOP JOIN / join op: CROSS JOIN`.
+
+That build side is **real, not a bad estimate**:
+
+| Query | HASH_JOIN requested | Decomposition |
+|---|---|---|
+| q08 | 537,032,000,000 B | ÷4 = 134,258 × 10⁶ = filtered_part × supplier |
+| q09 | 2,694,604,000,000 B | ÷4 = 673,651 × 10⁶ = filtered_part × supplier |
+
+The `× 10⁶` is simply SF100's supplier row count. The join OOMs after 100 retries, and
+`engine.rs`'s blanket `parked.clear()` then wipes every parked sender output — so the FE reported
+the collateral `no parked sender output to export for SenderSlot` error and the real cause was
+discarded. (That masking is fixed separately; the wipe now records and reports the true cause.)
+
+Reordering so every adjacent pair shares a predicate removes the cross join entirely: 0 NESTLOOP,
+7 HASH JOIN for q08. **No session variables are required** — with `cardinality: 1` everywhere the
+CBO has nothing to reorder with, so it follows the written order.
+
+### Measured (2× RTX PRO 6000, 2 CNs, SF100)
+
+| Query | Before | After | vs DuckDB oracle |
+|---|---|---|---|
+| q08 | never completed | **1897 ms** | MATCH, max rel diff 3.7e-05 |
+| q09 | never completed | **2115 ms** | 175/175 rows, all LOW by ~0.147 % (the known decimal-lowering defect) |
+
+Full sweep went from 19/22 to **20/22**, no timing regression (−0.4 % total).
+
+### The principled fix
+
+Give the CBO real cardinalities — then the stock text plans correctly and this deviation can be
+reverted. Options not yet evaluated: `ANALYZE` on the external scans, an external catalog with
+statistics, or injected stats. All may require a load step, which would break the benchmark's
+"read parquet directly, no load" property. Until then this reorder is the documented workaround.