Run

Run - mongodb-cdc-cayenne - changes #713

Workflow file for this run

.github/workflows/run_spicebench.yml at b4dc6a2

	name: Run
	run-name: Run - ${{ github.event.inputs.system_under_test \|\| 'spice_cloud' }} - ${{ github.event.inputs.etl_type}}

	on:
	schedule:
	- cron: '0 6 * * *' # Daily SF1.0 SCP run
	workflow_dispatch:
	inputs:
	scenario:
	description: 'Scenario/query set to run'
	required: true
	default: 'tpch'
	type: choice
	options:
	- tpch
	system_under_test:
	description: 'System under test (spice_cloud via spidapter docker image, local databricks adapter modes, or postgres via spidapter local backend)'
	required: true
	default: spice_cloud
	type: choice
	options:
	- spice_cloud
	- databricks-sql
	- databricks-lakebase
	- postgres-cdc-cayenne
	- postgres-cdc-duckdb
	- mongodb-cdc-cayenne
	- mongodb-cdc-duckdb
	- dynamodb-cdc-cayenne
	- dynamodb-cdc-duckdb
	etl_type:
	description: 'ETL type'
	required: true
	default: 'changes'
	type: choice
	options:
	- events
	- changes
	scale_factor:
	description: 'Scale Factor'
	required: true
	default: '1'
	type: choice
	options:
	- '0.1'
	- '1'
	- '10'

	env:
	SCENARIO: ${{ github.event.inputs.scenario \|\| 'tpch' }}
	SYSTEM_UNDER_TEST: ${{ github.event.inputs.system_under_test \|\| 'spice_cloud' }}
	ETL_TYPE: ${{ github.event.inputs.etl_type \|\| 'changes' }}
	SCALE_FACTOR: ${{ github.event.inputs.scale_factor \|\| '1' }}

	jobs:
	run-spicebench:
	name: Run spicebench
	runs-on: spiceai-dev-runners
	timeout-minutes: 600
	steps:
	- uses: actions/checkout@v6

	- uses: ./.github/actions/setup-cc

	- uses: ./.github/actions/management-login
	if: ${{ env.SYSTEM_UNDER_TEST == 'spice_cloud' \|\| startsWith(env.SYSTEM_UNDER_TEST, 'postgres-') }}
	with:
	token-url: https://spice.ai/api/oauth/token
	client-id: ${{ secrets.SPICE_MANAGEMENT_CLIENT_ID_PROD }}
	client-secret: ${{ secrets.SPICE_MANAGEMENT_CLIENT_SECRET_PROD }}

	- name: Log in to GHCR
	if: ${{ env.SYSTEM_UNDER_TEST == 'spice_cloud' \|\| startsWith(env.SYSTEM_UNDER_TEST, 'postgres-') }}
	uses: docker/login-action@v3
	with:
	registry: ghcr.io
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: pull spidapter image
	if: ${{ env.SYSTEM_UNDER_TEST == 'spice_cloud' \|\| startsWith(env.SYSTEM_UNDER_TEST, 'postgres-') }}
	run: docker pull ghcr.io/spiceai/spidapter:latest

	- uses: ./.github/actions/build-spicebench

	- name: Restore databricks adapter cache
	if: ${{ startsWith(env.SYSTEM_UNDER_TEST, 'databricks-') }}
	id: cache-databricks-adapter
	uses: actions/cache/restore@v4
	with:
	path: ~/.spice/bin/databricks-system-adapter
	key: databricks-system-adapter-${{ runner.os }}-${{ hashFiles('system-adapters/databricks/Cargo.toml', 'system-adapters/databricks/Cargo.lock', 'system-adapters/databricks/src/*/.rs', 'crates/system-adapter-protocol/Cargo.toml', 'crates/system-adapter-protocol/src/*/.rs') }}
	restore-keys: \|
	databricks-system-adapter-${{ runner.os }}-

	- name: Build databricks adapter
	if: ${{ startsWith(env.SYSTEM_UNDER_TEST, 'databricks-') && steps.cache-databricks-adapter.outputs.cache-hit != 'true' }}
	id: build-databricks-adapter
	run: \|
	mkdir -p ~/.spice/bin
	cargo build --manifest-path system-adapters/databricks/Cargo.toml
	install -m 755 system-adapters/databricks/target/debug/databricks-system-adapter ~/.spice/bin/databricks-system-adapter

	- name: Save databricks adapter cache
	if: ${{ startsWith(env.SYSTEM_UNDER_TEST, 'databricks-') && steps.build-databricks-adapter.outcome == 'success' }}
	uses: actions/cache/save@v4
	with:
	path: ~/.spice/bin/databricks-system-adapter
	key: databricks-system-adapter-${{ runner.os }}-${{ hashFiles('system-adapters/databricks/Cargo.toml', 'system-adapters/databricks/Cargo.lock', 'system-adapters/databricks/src/*/.rs', 'crates/system-adapter-protocol/Cargo.toml', 'crates/system-adapter-protocol/src/*/.rs') }}

	- name: Validate adapter configuration
	env:
	SPICEAI_API_KEY: ${{ env.SPICEAI_API_KEY }}
	SPICE_CLOUD_API_URL: https://api.spice.ai
	DATABRICKS_ENDPOINT: ${{ secrets.DATABRICKS_ENDPOINT }}
	DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
	DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }}
	DATABRICKS_SQL_WAREHOUSE_ID: ${{ secrets.DATABRICKS_SQL_WAREHOUSE_ID }}
	DATABRICKS_CATALOG: ${{ secrets.DATABRICKS_CATALOG }}
	DATABRICKS_SCHEMA: ${{ secrets.DATABRICKS_SCHEMA }}
	run: \|
	set -euo pipefail
	SYSTEM_UNDER_TEST_PREFIX="${SYSTEM_UNDER_TEST%%-*}"

	if [ -z "${SCENARIO}" ]; then
	echo "SCENARIO must not be empty"
	exit 1
	fi

	case "${SYSTEM_UNDER_TEST_PREFIX}" in
	spice_cloud)
	if [ -z "${SPICEAI_API_KEY:-}" ]; then
	echo "SPICEAI_API_KEY must be set for spice_cloud"
	exit 1
	fi

	if ! command -v docker >/dev/null 2>&1; then
	echo "docker is required for spice_cloud mode"
	exit 1
	fi

	docker image inspect ghcr.io/spiceai/spidapter:latest >/dev/null 2>&1 \|\| {
	echo "spidapter docker image not found locally; pull step may have failed"
	exit 1
	}
	;;

	databricks)
	for required_var in DATABRICKS_ENDPOINT DATABRICKS_TOKEN DATABRICKS_HTTP_PATH DATABRICKS_SQL_WAREHOUSE_ID; do
	if [ -z "${!required_var:-}" ]; then
	echo "${required_var} must be set for databricks adapter mode"
	exit 1
	fi
	done

	if echo "${DATABRICKS_ENDPOINT}" \| grep -qE '^https?://'; then
	echo "DATABRICKS_ENDPOINT should be a hostname only (no http/https scheme)"
	exit 1
	fi

	if echo "${DATABRICKS_HTTP_PATH}" \| grep -qE '^/'; then
	echo "DATABRICKS_HTTP_PATH should not start with '/'"
	exit 1
	fi

	if [ ! -x "${HOME}/.spice/bin/databricks-system-adapter" ]; then
	echo "Local databricks adapter binary is missing or not executable at ${HOME}/.spice/bin/databricks-system-adapter"
	exit 1
	fi

	"${HOME}/.spice/bin/databricks-system-adapter" --help >/dev/null
	;;

	postgres)
	if ! command -v docker >/dev/null 2>&1; then
	echo "docker is required for postgres mode"
	exit 1
	fi

	docker image inspect ghcr.io/spiceai/spidapter:latest >/dev/null 2>&1 \|\| {
	echo "spidapter docker image not found locally; pull step may have failed"
	exit 1
	}
	;;

	mongodb)
	if ! command -v docker >/dev/null 2>&1; then
	echo "docker is required for postgres mode"
	exit 1
	fi

	docker image inspect ghcr.io/spiceai/spidapter:latest >/dev/null 2>&1 \|\| {
	echo "spidapter docker image not found locally; pull step may have failed"
	exit 1
	}
	;;

	dynamodb)
	if ! command -v docker >/dev/null 2>&1; then
	echo "docker is required for postgres mode"
	exit 1
	fi

	docker image inspect ghcr.io/spiceai/spidapter:latest >/dev/null 2>&1 \|\| {
	echo "spidapter docker image not found locally; pull step may have failed"
	exit 1
	}
	;;

	*)
	echo "Unsupported system_under_test value: ${SYSTEM_UNDER_TEST}"
	exit 1
	;;
	esac

	- name: Setup Go
	if: ${{ startsWith(env.SYSTEM_UNDER_TEST, 'databricks-') }}
	uses: actions/setup-go@v5
	with:
	go-version: '1.23'

	- name: Checkout adbc-databricks Go driver
	if: ${{ startsWith(env.SYSTEM_UNDER_TEST, 'databricks-') }}
	uses: actions/checkout@v6
	with:
	repository: spiceai/adbc-databricks
	ref: spicebench
	path: adbc-databricks

	- name: Build databricks Go ADBC driver
	if: ${{ startsWith(env.SYSTEM_UNDER_TEST, 'databricks-') }}
	run: \|
	cd adbc-databricks/go
	go build -tags driverlib -buildmode=c-shared \
	-o build/libadbc_driver_databricks.so \
	./pkg/
	sudo install -m 755 build/libadbc_driver_databricks.so /usr/local/lib/libdatabricks.so
	sudo ldconfig

	- name: Install ADBC Postgres driver
	if: ${{ startsWith(env.SYSTEM_UNDER_TEST, 'databricks-') \|\| startsWith(env.SYSTEM_UNDER_TEST, 'postgres-') }}
	uses: columnar-tech/setup-dbc@v1
	with:
	drivers: postgresql

	- name: Install ADBC FlightSQL driver
	if: ${{ !startsWith(env.SYSTEM_UNDER_TEST, 'databricks-') }}
	uses: columnar-tech/setup-dbc@v1
	with:
	drivers: flightsql

	- name: Run spicebench
	env:
	SPICEAI_API_KEY: ${{ env.SPICEAI_API_KEY }}
	SPICE_CLOUD_API_URL: https://api.spice.ai
	DATABRICKS_ENDPOINT: ${{ secrets.DATABRICKS_ENDPOINT }}
	DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
	DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }}
	DATABRICKS_SQL_WAREHOUSE_ID: ${{ secrets.DATABRICKS_SQL_WAREHOUSE_ID }}
	DATABRICKS_CATALOG: ${{ vars.DATABRICKS_CATALOG }}
	DATABRICKS_SCHEMA: ${{ vars.DATABRICKS_SCHEMA }}
	DATABRICKS_STAGING_VOLUME_PATH: ${{ vars.DATABRICKS_STAGING_VOLUME_PATH }}
	SYSTEM_ADAPTER: ${{ env.SYSTEM_UNDER_TEST }}
	NUM_QUERY_CLIENTS: '2'
	ETL_BUCKET: 'spicebench'
	ETL_PREFIX: ${{ env.ETL_TYPE == 'changes' && 'data-gen-mutable' \|\| 'data-gen' }}
	ETL_REGION: 'us-east-1'
	ETL_SINK: 'adbc'
	SCHEDULER_STATE_LOCATION: 's3://spiceai-testing-cluster-state/spicebench-scheduler-state-${{ github.run_id }}/'
	VALIDATE_CHECKPOINT_RESULTS: 'true'
	ENABLE_MODULE_DEBUG_LOGGING: 'false'
	SCRAPE_SUT_METRICS: 'true'
	SPICEAI_BENCHMARK_METRICS_KEY: ${{ secrets.SPICEAI_BENCHMARK_METRICS_KEY }}
	MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }}
	AWS_ACCESS_KEY_ID: ${{ secrets.MINIO_ACCESS_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.MINIO_SECRET_ACCESS_KEY }}
	S3_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
	S3_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	LAKEBASE_PG_HOST: ${{ vars.LAKEBASE_PG_HOST }}
	LAKEBASE_PG_USER: ${{ vars.LAKEBASE_PG_USER }}
	LAKEBASE_PG_SCHEMA: ${{ vars.LAKEBASE_PG_SCHEMA }}
	LAKEBASE_PROJECT: ${{ vars.LAKEBASE_PROJECT }}
	LAKEBASE_BRANCH: ${{ vars.LAKEBASE_BRANCH }}
	EC2_SUBNET_ID: ${{ vars.EC2_SUBNET_ID }}
	EC2_SECURITY_GROUP_ID: ${{ vars.EC2_SECURITY_GROUP_ID }}
	EC2_AMI_ID: ${{ vars.EC2_AMI_ID }}
	EC2_INSTANCE_TYPE: ${{ vars.EC2_INSTANCE_TYPE }}
	EC2_IAM_INSTANCE_PROFILE: ${{ vars.EC2_IAM_INSTANCE_PROFILE }}
	SPIDAPTER_ICEBERG_REGION: us-west-1
	SPIDAPTER_ICEBERG_CATALOG_FROM: iceberg:https://glue.us-west-1.amazonaws.com/iceberg/v1/catalogs/211125479522/namespaces
	SPIDAPTER_APP_MEMORY_LIMIT: '62Gi'
	SPIDAPTER_EXECUTOR_MEMORY_LIMIT: '62Gi'
	SPIDAPTER_EPHEMERAL_STORAGE_LIMIT_GB: '256Gi'
	ENABLE_PVC: 'false'
	run: \|
	set -euo pipefail
	if [ "${ENABLE_MODULE_DEBUG_LOGGING}" = "true" ]; then
	export RUST_LOG='info,etl=debug,spicebench=debug,data_generation=debug'
	else
	export RUST_LOG='info,etl::sink::adbc=debug'
	fi

	TABLE_FORMAT="parquet"
	EXECUTOR_INSTANCE_TYPE="github-hosted-ubuntu-latest"
	ETL_ENDPOINT="${MINIO_ENDPOINT}"
	DATABRICKS_TABLE_FORMAT="${TABLE_FORMAT}"
	SYSTEM_UNDER_TEST_PREFIX="${SYSTEM_UNDER_TEST%%-*}"
	ETL_ARGS="--etl-bucket ${ETL_BUCKET} --scale-factor ${SCALE_FACTOR}"
	if [ -n "${ETL_PREFIX}" ]; then
	ETL_ARGS="${ETL_ARGS} --etl-prefix ${ETL_PREFIX}"
	fi
	if [ -n "${ETL_REGION}" ]; then
	ETL_ARGS="${ETL_ARGS} --etl-region ${ETL_REGION}"
	fi
	if [ -n "${ETL_ENDPOINT:-}" ]; then
	ETL_ARGS="${ETL_ARGS} --etl-endpoint ${ETL_ENDPOINT}"
	fi

	ETL_SINK_ARGS="--etl-sink ${ETL_SINK} --table-format ${TABLE_FORMAT}"
	if [ "${ETL_SINK}" = "adbc" ]; then
	:
	fi

	VALIDATION_ARGS=""
	if [ "${VALIDATE_CHECKPOINT_RESULTS}" = "true" ]; then
	VALIDATION_ARGS="--validate-results"
	fi

	SCHEDULER_STATE_ADAPTER_ENV="--system-adapter-env SCHEDULER_STATE_LOCATION=${SCHEDULER_STATE_LOCATION}"

	SUT_METRICS_ARGS=""
	if [ "${SCRAPE_SUT_METRICS}" = "true" ]; then
	SUT_METRICS_ARGS="--scrape-sut-metrics"
	fi

	if [ "${SYSTEM_UNDER_TEST_PREFIX}" = "databricks" ]; then
	export SPICEBENCH_ADBC_DELETE_BATCH_SIZE=50000
	export SPICEBENCH_ADBC_UPDATE_STRATEGY=staging_table
	export SPICEBENCH_TARGET_BATCH_ROWS=500000
	export SPICEBENCH_ADBC_MAX_INGEST_BATCH_BYTES=1268435456
	export SPICEBENCH_ADBC_REUSE_BULK_INGEST_STREAMS=false
	ADAPTER_CMD="${HOME}/.spice/bin/databricks-system-adapter"
	ADAPTER_ARGS="stdio"
	ADAPTER_ENVS="--system-adapter-env DATABRICKS_ENDPOINT=${DATABRICKS_ENDPOINT} --system-adapter-env DATABRICKS_TOKEN=${DATABRICKS_TOKEN} --system-adapter-env DATABRICKS_HTTP_PATH=${DATABRICKS_HTTP_PATH} --system-adapter-env DATABRICKS_SQL_WAREHOUSE_ID=${DATABRICKS_SQL_WAREHOUSE_ID} --system-adapter-env DATABRICKS_TABLE_FORMAT=${DATABRICKS_TABLE_FORMAT}"

	if [ -n "${DATABRICKS_CATALOG:-}" ]; then
	ADAPTER_ENVS="${ADAPTER_ENVS} --system-adapter-env DATABRICKS_CATALOG=${DATABRICKS_CATALOG}"
	fi

	if [ -n "${DATABRICKS_SCHEMA:-}" ]; then
	ADAPTER_ENVS="${ADAPTER_ENVS} --system-adapter-env DATABRICKS_SCHEMA=${DATABRICKS_SCHEMA}"
	fi

	if [ -n "${DATABRICKS_STAGING_VOLUME_PATH:-}" ]; then
	ADAPTER_ENVS="${ADAPTER_ENVS} --system-adapter-env DATABRICKS_STAGING_VOLUME_PATH=${DATABRICKS_STAGING_VOLUME_PATH}"
	fi

	if [ "${SYSTEM_UNDER_TEST}" = "databricks-lakebase" ]; then
	ADAPTER_ENVS="${ADAPTER_ENVS} --system-adapter-env DATABRICKS_COMPUTE_MODE=lakebase"
	ADAPTER_ENVS="${ADAPTER_ENVS} --system-adapter-env LAKEBASE_PG_HOST=${LAKEBASE_PG_HOST}"
	ADAPTER_ENVS="${ADAPTER_ENVS} --system-adapter-env LAKEBASE_PG_USER=${LAKEBASE_PG_USER}"
	ADAPTER_ENVS="${ADAPTER_ENVS} --system-adapter-env LAKEBASE_PG_SCHEMA=${LAKEBASE_PG_SCHEMA}"
	ADAPTER_ENVS="${ADAPTER_ENVS} --system-adapter-env LAKEBASE_PROJECT=${LAKEBASE_PROJECT}"
	ADAPTER_ENVS="${ADAPTER_ENVS} --system-adapter-env LAKEBASE_BRANCH=${LAKEBASE_BRANCH}"
	fi
	elif [ "${SYSTEM_UNDER_TEST_PREFIX}" = "postgres" ]; then
	export SPICEBENCH_TARGET_BATCH_ROWS=50000
	export SPICEBENCH_ADBC_DELETE_BATCH_SIZE=1000
	export SPICEBENCH_ADBC_UPDATE_STRATEGY=staging_table
	case "${SYSTEM_UNDER_TEST}" in
	postgres-cdc-duckdb) SPICE_ACCELERATION="duckdb" ;;
	*) SPICE_ACCELERATION="cayenne" ;;
	esac
	ADAPTER_CMD="docker"
	ADAPTER_DOCKER_OPTS="run -i -e SPICE_ACCELERATION=${SPICE_ACCELERATION} -e EC2_SUBNET_ID -e EC2_SECURITY_GROUP_ID -e EC2_AMI_ID -e EC2_INSTANCE_TYPE -e AWS_REGION=us-west-2 -e EC2_IAM_INSTANCE_PROFILE"
	ADAPTER_DOCKER_OPTS="${ADAPTER_DOCKER_OPTS} -e AWS_ACCESS_KEY_ID=${S3_AWS_ACCESS_KEY_ID} -e AWS_SECRET_ACCESS_KEY=${S3_AWS_SECRET_ACCESS_KEY}"
	ADAPTER_DOCKER_OPTS="${ADAPTER_DOCKER_OPTS} -e SPICEAI_API_KEY -e SPICE_CLOUD_API_URL"
	ADAPTER_ARGS="${ADAPTER_DOCKER_OPTS} ghcr.io/spiceai/spidapter:latest stdio --compute cloud --storage postgres --verbose --channel nightly --ec2-associate-public-ip"
	ADAPTER_ENVS=""
	elif [ "${SYSTEM_UNDER_TEST_PREFIX}" = "mongodb" ]; then
	export SPICEBENCH_TARGET_BATCH_ROWS=50000
	export SPICEBENCH_ADBC_DELETE_BATCH_SIZE=1000
	export SPICEBENCH_ADBC_UPDATE_STRATEGY=statement
	case "${SYSTEM_UNDER_TEST}" in
	mongodb-cdc-duckdb) SPICE_ACCELERATION="duckdb" ;;
	*) SPICE_ACCELERATION="cayenne" ;;
	esac
	ADAPTER_CMD="docker"
	ADAPTER_DOCKER_OPTS="run -i -e SPICE_ACCELERATION=${SPICE_ACCELERATION} -e EC2_SUBNET_ID -e EC2_SECURITY_GROUP_ID -e EC2_AMI_ID -e EC2_INSTANCE_TYPE -e AWS_REGION=us-west-2 -e EC2_IAM_INSTANCE_PROFILE"
	ADAPTER_DOCKER_OPTS="${ADAPTER_DOCKER_OPTS} -e AWS_ACCESS_KEY_ID=${S3_AWS_ACCESS_KEY_ID} -e AWS_SECRET_ACCESS_KEY=${S3_AWS_SECRET_ACCESS_KEY}"
	ADAPTER_DOCKER_OPTS="${ADAPTER_DOCKER_OPTS} -e SPICEAI_API_KEY -e SPICE_CLOUD_API_URL"
	ADAPTER_ARGS="${ADAPTER_DOCKER_OPTS} ghcr.io/spiceai/spidapter:latest stdio --compute cloud --storage mongo --verbose --channel nightly --ec2-associate-public-ip"
	ADAPTER_ENVS=""
	elif [ "${SYSTEM_UNDER_TEST_PREFIX}" = "dynamodb" ]; then
	export SPICEBENCH_TARGET_BATCH_ROWS=50000
	export SPICEBENCH_ADBC_DELETE_BATCH_SIZE=1000
	export SPICEBENCH_ADBC_UPDATE_STRATEGY=bulk_ingest_upsert
	case "${SYSTEM_UNDER_TEST}" in
	dynamodb-cdc-duckdb) SPICE_ACCELERATION="duckdb" ;;
	*) SPICE_ACCELERATION="cayenne" ;;
	esac
	ADAPTER_CMD="docker"
	ADAPTER_DOCKER_OPTS="run -i -e SPICE_ACCELERATION=${SPICE_ACCELERATION}"
	ADAPTER_DOCKER_OPTS="${ADAPTER_DOCKER_OPTS} -e AWS_ACCESS_KEY_ID=${S3_AWS_ACCESS_KEY_ID} -e AWS_SECRET_ACCESS_KEY=${S3_AWS_SECRET_ACCESS_KEY}"
	ADAPTER_DOCKER_OPTS="${ADAPTER_DOCKER_OPTS} -e SPICEAI_API_KEY -e SPICE_CLOUD_API_URL"
	ADAPTER_ARGS="${ADAPTER_DOCKER_OPTS} ghcr.io/spiceai/spidapter:latest stdio --compute cloud --storage dynamodb --verbose --channel nightly"
	ADAPTER_ENVS=""
	else
	export SPICEBENCH_ADBC_UPDATE_STRATEGY=bulk_ingest_upsert
	export SPICEBENCH_ADBC_FLUSH_STREAM_BEFORE_UPSERT=true
	export SPICEBENCH_ADBC_DELETE_BATCH_SIZE=50000
	ADAPTER_CMD="docker"
	ADAPTER_DOCKER_OPTS="run -i -e SPIDAPTER_EXECUTOR_REPLICAS=4 -e SPICEAI_API_KEY -e SPICE_CLOUD_API_URL -e AWS_ACCESS_KEY_ID=${S3_AWS_ACCESS_KEY_ID} -e AWS_SECRET_ACCESS_KEY=${S3_AWS_SECRET_ACCESS_KEY} -e SPIDAPTER_ICEBERG_REGION -e SPIDAPTER_ICEBERG_CATALOG_FROM -e SCHEDULER_STATE_LOCATION"

	ADAPTER_DOCKER_OPTS="${ADAPTER_DOCKER_OPTS} -e SPIDAPTER_APP_MEMORY_LIMIT=${SPIDAPTER_APP_MEMORY_LIMIT}"
	ADAPTER_DOCKER_OPTS="${ADAPTER_DOCKER_OPTS} -e SPIDAPTER_EXECUTOR_MEMORY_LIMIT=${SPIDAPTER_EXECUTOR_MEMORY_LIMIT}"
	ADAPTER_DOCKER_OPTS="${ADAPTER_DOCKER_OPTS} -e SPIDAPTER_EPHEMERAL_STORAGE_LIMIT_GB=${SPIDAPTER_EPHEMERAL_STORAGE_LIMIT_GB}"
	if [ "${ENABLE_PVC}" = "true" ]; then
	echo "PVC enabled: app=3GB, executor=2GB"
	ADAPTER_DOCKER_OPTS="${ADAPTER_DOCKER_OPTS} -e SPIDAPTER_APP_STORAGE_SIZE_GB=3 -e SPIDAPTER_EXECUTOR_STORAGE_SIZE_GB=2 -e SPIDAPTER_CAYENNE_DATA_DIR=/data/data -e SPIDAPTER_CAYENNE_METADATA_DIR=/data/metadata"
	fi

	ADAPTER_ARGS="${ADAPTER_DOCKER_OPTS} ghcr.io/spiceai/spidapter:latest stdio --verbose --channel nightly"
	ADAPTER_ENVS=""
	fi

	set -x
	~/.spice/bin/spicebench run \
	--concurrency "${NUM_QUERY_CLIENTS}" \
	--scenario "${SCENARIO}" \
	--executor-instance-type "${EXECUTOR_INSTANCE_TYPE}" \
	${ETL_ARGS} \
	${ETL_SINK_ARGS} \
	${VALIDATION_ARGS} \
	${SUT_METRICS_ARGS} \
	--system-adapter-stdio-cmd "${ADAPTER_CMD}" \
	--system-adapter-stdio-args "${ADAPTER_ARGS}" \
	${ADAPTER_ENVS} \
	${SCHEDULER_STATE_ADAPTER_ENV}
	set +x

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Run - mongodb-cdc-cayenne - changes #713

Workflow file

Run - mongodb-cdc-cayenne - changes #713

Uh oh!

Workflow file for this run