always run on spiceai-dev-runners, use minio for storage

Author: peasee

.github/workflows/data_generation_run.yml

@@ -32,14 +32,12 @@ on:
         required: false
         default: '100'
         type: string
-      runner_type:
-        required: false
-        default: 'ubuntu-latest'
-        type: string
     secrets:
-      AWS_ACCESS_KEY_ID:
+      MINIO_ENDPOINT:
+        required: true
+      MINIO_ACCESS_KEY_ID:
         required: true
-      AWS_SECRET_ACCESS_KEY:
+      MINIO_SECRET_ACCESS_KEY:
         required: true
   workflow_dispatch:
     inputs:
@@ -48,11 +46,6 @@ on:
         required: true
         default: 'tpch'
         type: string
-      runner_type:
-        description: 'GitHub runner label to execute on'
-        required: false
-        default: 'spiceai-macos'
-        type: string
       scale_factor:
         description: 'TPC-H scale factor'
         required: true
@@ -87,7 +80,7 @@ on:
 jobs:
   run-data-generation:
     name: Run data generation
-    runs-on: ${{ inputs.runner_type || github.event.inputs.runner_type || 'ubuntu-latest' }}
+    runs-on: spiceai-dev-runners
     timeout-minutes: 600
     steps:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
@@ -123,8 +116,9 @@ jobs:
           PREFIX: ${{ inputs.prefix || github.event.inputs.prefix || 'data-gen' }}
           REGION: ${{ inputs.region || github.event.inputs.region || 'us-east-1' }}
           NUM_STEPS: ${{ inputs.num_steps || github.event.inputs.num_steps || '25' }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.MINIO_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.MINIO_SECRET_ACCESS_KEY }}
           RUST_LOG: info
         run: |
           ARGS="--dataset ${SCENARIO}"
@@ -134,6 +128,7 @@ jobs:
           ARGS="${ARGS} --prefix ${PREFIX}"
           ARGS="${ARGS} --region ${REGION}"
           ARGS="${ARGS} --num-steps ${NUM_STEPS}"
+          ARGS="${ARGS} --endpoint ${MINIO_ENDPOINT}"
 
           echo "Running: data-generation run ${ARGS}"
           ~/.spice/bin/data-generation run ${ARGS}
@@ -162,8 +157,9 @@ jobs:
           PREFIX: ${{ inputs.prefix || github.event.inputs.prefix || 'data-gen' }}
           REGION: ${{ inputs.region || github.event.inputs.region || 'us-east-1' }}
           CHECKPOINT_INTERVAL_STEPS: ${{ inputs.checkpoint_interval_steps || github.event.inputs.checkpoint_interval_steps || '100' }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.MINIO_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.MINIO_SECRET_ACCESS_KEY }}
           RUST_LOG: info
         run: |
           VERSION=$(python3 - <<'PY'
@@ -191,6 +187,7 @@ jobs:
           fi
 
           ARGS="${ARGS} --checkpoint-interval-steps ${CHECKPOINT_INTERVAL_STEPS}"
+          ARGS="${ARGS} --endpoint ${MINIO_ENDPOINT}"
 
           echo "Running: checkpointer ${ARGS}"
           ~/.spice/bin/checkpointer ${ARGS}

.github/workflows/run_spicebench.yml

@@ -53,14 +53,6 @@ on:
         required: false
         default: false
         type: boolean
-      runner:
-        description: 'GitHub runner label to use for this workflow run'
-        required: true
-        default: ubuntu-latest
-        type: choice
-        options:
-          - ubuntu-latest
-          - spiceai-dev-runners
       scheduler_state_location:
         description: 'S3 URI for shared scheduler state (e.g. s3://bucket/scheduler-state/). If empty, scheduler state is not configured.'
         required: false
@@ -70,7 +62,7 @@ on:
 jobs:
   run-spicebench:
     name: Run spicebench
-    runs-on: ${{ github.event.inputs.runner || 'ubuntu-latest' }}
+    runs-on: spiceai-dev-runners
     timeout-minutes: 600
     steps:
       - uses: actions/checkout@v6
@@ -222,19 +214,15 @@ jobs:
         run: |
           set -euo pipefail
           curl -LsSf https://dbc.columnar.tech/install.sh | sh
-          if [ "${{ github.event.inputs.runner || 'ubuntu-latest' }}" = "spiceai-dev-runners" ]; then
-            source "$HOME/.local/bin/env"
-          fi
+          source "$HOME/.local/bin/env"
           dbc install postgresql
 
       - name: Install ADBC FlightSQL driver
         if: ${{ !startsWith(github.event.inputs.system_under_test || 'spice_cloud', 'databricks-') }}
         run: |
           set -euo pipefail
           curl -LsSf https://dbc.columnar.tech/install.sh | sh
-          if [ "${{ github.event.inputs.runner || 'ubuntu-latest' }}" = "spiceai-dev-runners" ]; then
-            source "$HOME/.local/bin/env"
-          fi
+          source "$HOME/.local/bin/env"
           dbc install flightsql
 
       - name: Run spicebench
@@ -261,8 +249,9 @@ jobs:
           VALIDATE_CHECKPOINT_RESULTS: ${{ github.event.inputs.validate_checkpoint_results || 'false' }}
           ENABLE_MODULE_DEBUG_LOGGING: ${{ github.event.inputs.enable_module_debug_logging || 'false' }}
           SPICEAI_BENCHMARK_METRICS_KEY: ${{ secrets.SPICEAI_BENCHMARK_METRICS_KEY }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.MINIO_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.MINIO_SECRET_ACCESS_KEY }}
           LAKEBASE_PG_HOST: ${{ secrets.LAKEBASE_PG_HOST }}
           LAKEBASE_PG_USER: ${{ secrets.LAKEBASE_PG_USER }}
           LAKEBASE_PG_DB_NAME: ${{ secrets.LAKEBASE_PG_DB_NAME }}
@@ -280,7 +269,7 @@ jobs:
 
           TABLE_FORMAT="parquet"
           EXECUTOR_INSTANCE_TYPE="github-hosted-ubuntu-latest"
-          ETL_ENDPOINT=""
+          ETL_ENDPOINT="${MINIO_ENDPOINT}"
           DATABRICKS_TABLE_FORMAT="${TABLE_FORMAT}"
           SYSTEM_UNDER_TEST_PREFIX="${SYSTEM_UNDER_TEST%%-*}"
           ETL_ARGS="--etl-bucket ${ETL_BUCKET} --scale-factor ${SCALE_FACTOR}"