feat(postgres): stream WAL directly to Spice accelerators (#10364) #3729
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: integration tests | |
| on: | |
| push: | |
| branches: | |
| - trunk | |
| - release/* | |
| pull_request: | |
| branches: | |
| - trunk | |
| - release-* | |
| - release/* | |
| - feature-* | |
| merge_group: | |
| branches: | |
| - trunk | |
| - release-* | |
| - release/* | |
| - feature-* | |
| workflow_dispatch: | |
| inputs: | |
| update_snapshots: | |
| description: 'Update the snapshots?' | |
| required: false | |
| default: 'no' | |
| type: choice | |
| options: | |
| - 'always' | |
| - 'no' | |
| run_all_tests: | |
| description: 'Run all tests' | |
| type: boolean | |
| required: false | |
| default: false | |
| concurrency: | |
| # Allow only one workflow per any non-trunk branch. | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_name == 'trunk' && github.sha || 'any-sha' }} | |
| cancel-in-progress: true | |
| env: | |
| CONTAINER_REGISTRY: spiceaitestimages.azurecr.io/ | |
| # CI performance optimizations | |
| CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse | |
| CARGO_NET_RETRY: 10 | |
| CARGO_HTTP_TIMEOUT: 60 | |
| CARGO_INCREMENTAL: 0 | |
| CARGO_NET_GIT_FETCH_WITH_CLI: true | |
| jobs: | |
| # `pr` is a required check for pull requests. Therefore we cannot skip the workflow via | |
| # `on.pull_requests.paths-ignore`. Instead, we conditionally run the jobs (which report | |
| # a success when skipped). | |
| check_changes: | |
| runs-on: spiceai-dev-runners | |
| outputs: | |
| relevant_changes: ${{ steps.check.outputs.relevant_changes }} | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 | |
| with: | |
| # Use fetch-depth 2 for PR change detection (dorny/paths-filter only needs base + head) | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }} | |
| - name: Check for code changes | |
| id: check | |
| uses: ./.github/actions/check-code-changes | |
| build: | |
| name: Build Test Binary | |
| needs: check_changes | |
| runs-on: spiceai-dev-runners | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| with: | |
| fetch-depth: 1 | |
| # Cache Cargo dependencies | |
| - name: Cache Cargo registry | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 | |
| with: | |
| path: | | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo- | |
| - name: Set up Rust | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: ./.github/actions/setup-rust | |
| - name: Set up make | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: ./.github/actions/setup-make | |
| - name: Set up cc | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: ./.github/actions/setup-cc | |
| - name: Install protoc | |
| uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up sccache | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: ./.github/actions/setup-sccache | |
| with: | |
| minio_endpoint: ${{ secrets.TEST_MINIO_ENDPOINT }} | |
| - name: Set up Nextest | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: ./.github/actions/setup-nextest | |
| # Build the test binaries without running tests | |
| - name: Build integration test archives | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| env: | |
| CARGO_INCREMENTAL: 0 | |
| RUSTC_WRAPPER: sccache | |
| AWS_ACCESS_KEY_ID: ${{ secrets.TEST_MINIO_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_MINIO_SECRET_KEY }} | |
| run: | | |
| set -euo pipefail | |
| FEATURES="postgres,mysql,delta_lake,duckdb,sqlite,spark,databricks,dynamodb,oracle,kafka,iceberg-write,snapshots,mongodb" | |
| if [[ "${{ github.event.inputs.run_all_tests }}" == "true" ]]; then | |
| FEATURES="${FEATURES},extended_tests" | |
| echo "Including extended_tests" | |
| else | |
| echo "Excluding extended_tests" | |
| fi | |
| cargo nextest archive -p runtime --test integration --features ${FEATURES} --archive-file integration.tar.zst | |
| cargo nextest archive -p runtime --test retention_oom --features ${FEATURES} --archive-file retention_oom.tar.zst | |
| cargo nextest archive -p runtime --test integration_aws_sdk --features databricks,delta_lake --archive-file integration_aws_sdk.tar.zst | |
| cargo nextest archive -p aws-sdk-credential-bridge --test credential_provider --archive-file integration_aws_sdk_credential_bridge.tar.zst | |
| cargo nextest archive -p runtime-table-partition --test partition_table_provider --archive-file partition_table_test.tar.zst | |
| cargo nextest archive -p data_components --test hadoop_catalog_test --archive-file data_components_hadoop.tar.zst | |
| # Upload the test archive as an artifact | |
| - name: Upload test archive | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: integration-test-archive | |
| path: ./integration.tar.zst | |
| retention-days: 3 | |
| # Archive is already zstd-compressed; use minimal artifact zip compression. | |
| compression-level: 1 | |
| - name: Upload retention OOM test archive | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: integration-retention-oom-test-archive | |
| path: ./retention_oom.tar.zst | |
| retention-days: 3 | |
| # Archive is already zstd-compressed; use minimal artifact zip compression. | |
| compression-level: 1 | |
| - name: Upload AWS SDK test archive | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: integration-aws-sdk-test-archive | |
| path: ./integration_aws_sdk.tar.zst | |
| retention-days: 3 | |
| # Archive is already zstd-compressed; use minimal artifact zip compression. | |
| compression-level: 1 | |
| - name: Upload AWS SDK credential bridge test archive | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: integration-aws-sdk-credential-bridge-test-archive | |
| path: ./integration_aws_sdk_credential_bridge.tar.zst | |
| retention-days: 3 | |
| # Archive is already zstd-compressed; use minimal artifact zip compression. | |
| compression-level: 1 | |
| - name: Upload partition table test archive | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: integration-partition-table-test-archive | |
| path: ./partition_table_test.tar.zst | |
| retention-days: 3 | |
| # Archive is already zstd-compressed; use minimal artifact zip compression. | |
| compression-level: 1 | |
| - name: Upload data components test archive | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: integration-data-components-test-archive | |
| path: ./data_components_hadoop.tar.zst | |
| retention-days: 3 | |
| # Archive is already zstd-compressed; use minimal artifact zip compression. | |
| compression-level: 1 | |
| test: | |
| name: Integration Tests (part ${{ matrix.label }}) | |
| needs: [build, check_changes] | |
| permissions: | |
| contents: write | |
| runs-on: spiceai-dev-runners | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - label: 1 | |
| partition: 'count:1/3' | |
| - label: 2 | |
| partition: 'count:2/3' | |
| - label: 3 | |
| partition: 'count:3/3' | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| with: | |
| fetch-depth: 1 # Shallow clone for faster checkout | |
| - name: Set up Rust | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: ./.github/actions/setup-rust | |
| - name: Install Oracle ODPI-C | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: ./.github/actions/setup-oracle-odpi | |
| # Download the test archive artifact | |
| - name: Download test archive | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: integration-test-archive | |
| path: ./integration_test | |
| - name: Download retention OOM test archive | |
| if: needs.check_changes.outputs.relevant_changes == 'true' && matrix.partition == 'count:1/3' | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: integration-retention-oom-test-archive | |
| path: ./retention_oom_test | |
| - name: Set up Nextest | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: ./.github/actions/setup-nextest | |
| - name: Login to ACR | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 | |
| with: | |
| registry: spiceaitestimages.azurecr.io | |
| username: spiceai-repo-pull | |
| password: ${{ secrets.AZCR_PASSWORD }} | |
| - name: Pull the Postgres/MySQL images | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| run: | | |
| echo ${{ env.CONTAINER_REGISTRY }} | |
| docker pull ${{ env.CONTAINER_REGISTRY }}postgres:latest | |
| docker pull ${{ env.CONTAINER_REGISTRY }}mysql:latest | |
| - name: Set up Spice.ai API Key | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| run: | | |
| echo 'SPICEAI_API_KEY="${{ secrets.SPICE_SECRET_SPICEAI_KEY }}"' > .env | |
| - name: Set up Github Token | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v2 | |
| id: github-app-token | |
| with: | |
| app-id: ${{ vars.ORG_MEMBERS_GITHUB_APP_ID }} | |
| private-key: ${{ secrets.ORG_MEMBERS_GITHUB_PRIVATE_KEY }} | |
| owner: spiceai | |
| repositories: | | |
| spiceai | |
| - name: Run integration test | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| env: | |
| AWS_EC2_METADATA_DISABLED: true | |
| INSTA_UPDATE: ${{ github.event.inputs.update_snapshots }} | |
| SPICE_SECRET_SPICEAI_KEY: ${{ secrets.SPICE_SECRET_SPICEAI_KEY }} | |
| GITHUB_ORG_TOKEN: ${{ steps.github-app-token.outputs.token }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # MinIO/S3-compatible storage for TPC-DS and other S3 tests | |
| MINIO_ENDPOINT: ${{ secrets.TEST_MINIO_ENDPOINT }} | |
| MINIO_ACCESS_KEY_ID: ${{ secrets.TEST_MINIO_ACCESS_KEY }} | |
| MINIO_SECRET_ACCESS_KEY: ${{ secrets.TEST_MINIO_SECRET_KEY }} | |
| # old dbc instance | |
| TEST_DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }} | |
| TEST_DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} | |
| TEST_DATABRICKS_CLUSTER_ID: ${{ vars.DATABRICKS_CLUSTER_ID }} | |
| AWS_DATABRICKS_DELTA_ACCESS_KEY_ID: ${{ secrets.AWS_DATABRICKS_DELTA_ACCESS_KEY_ID }} | |
| AWS_DATABRICKS_DELTA_SECRET_ACCESS_KEY: ${{ secrets.AWS_DATABRICKS_DELTA_SECRET_ACCESS_KEY }} | |
| # new dbc instance | |
| NEW_DATABRICKS_HOST: ${{ vars.NEW_DATABRICKS_HOST }} | |
| NEW_DATABRICKS_CLUSTER_ID: ${{ vars.NEW_DATABRICKS_CLUSTER_ID }} | |
| NEW_DATABRICKS_SP_CLIENT_ID: ${{ secrets.NEW_DATABRICKS_SP_CLIENT_ID }} | |
| NEW_DATABRICKS_SP_CLIENT_SECRET: ${{ secrets.NEW_DATABRICKS_SP_CLIENT_SECRET }} | |
| NEW_AWS_DATABRICKS_DELTA_ACCESS_KEY_ID: ${{ secrets.NEW_AWS_DATABRICKS_DELTA_ACCESS_KEY_ID }} | |
| NEW_AWS_DATABRICKS_DELTA_SECRET_ACCESS_KEY: ${{ secrets.NEW_AWS_DATABRICKS_DELTA_SECRET_ACCESS_KEY }} | |
| SPICEAI_API_KEY: ${{ secrets.SPICE_SECRET_SPICEAI_TPCH_BENCHMARK_KEY }} | |
| AWS_ICEBERG_ACCESS_KEY_ID: ${{ secrets.AWS_ICEBERG_ACCESS_KEY_ID }} | |
| AWS_ICEBERG_SECRET_ACCESS_KEY: ${{ secrets.AWS_ICEBERG_SECRET_ACCESS_KEY }} | |
| AWS_ICEBERG_REGION: ${{ secrets.AWS_ICEBERG_REGION }} | |
| AWS_ICEBERG_ACCOUNT_ID: ${{ secrets.AWS_ICEBERG_ACCOUNT_ID }} | |
| AWS_GLUE_KEY: ${{ secrets.AWS_ICEBERG_ACCESS_KEY_ID }} | |
| AWS_GLUE_SECRET: ${{ secrets.AWS_ICEBERG_SECRET_ACCESS_KEY }} | |
| AWS_DYNAMODB_KEY: ${{ secrets.AWS_DYNAMODB_ACCESS_KEY_ID }} | |
| AWS_DYNAMODB_SECRET: ${{ secrets.AWS_DYNAMODB_SECRET_ACCESS_KEY }} | |
| ORACLE_CLOUD_CONNECTION_STRING: ${{ secrets.ORACLE_CLOUD_CONNECTION_STRING }} | |
| ORACLE_CLOUD_USERNAME: ${{ secrets.ORACLE_CLOUD_USERNAME }} | |
| ORACLE_CLOUD_PASSWORD: ${{ secrets.ORACLE_CLOUD_PASSWORD }} | |
| ORACLE_CLOUD_WALLET_SSO_CERT: ${{ secrets.ORACLE_CLOUD_WALLET_SSO_CERT }} | |
| AWS_SNAPSHOT_KEY: ${{ secrets.AWS_ICEBERG_ACCESS_KEY_ID }} | |
| AWS_SNAPSHOT_SECRET: ${{ secrets.AWS_ICEBERG_SECRET_ACCESS_KEY }} | |
| AWS_S3_VECTORS_KEY: ${{ secrets.AWS_S3_VECTORS_KEY }} | |
| AWS_S3_VECTORS_SECRET: ${{ secrets.AWS_S3_VECTORS_SECRET }} | |
| run: | | |
| if [ -n "$SPICE_SECRET_SPICEAI_KEY" ]; then | |
| INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --partition '${{ matrix.partition }}' --archive-file ./integration_test/integration.tar.zst | |
| else | |
| INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --partition '${{ matrix.partition }}' --archive-file ./integration_test/integration.tar.zst --skip spiceai_integration_test | |
| fi | |
| - name: Run retention OOM regression test | |
| if: needs.check_changes.outputs.relevant_changes == 'true' && matrix.partition == 'count:1/3' | |
| run: | | |
| INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --archive-file ./retention_oom_test/retention_oom.tar.zst | |
| - name: Upload integration snapshots artifact | |
| id: create_snapshot_archive | |
| if: github.event.inputs.update_snapshots == 'always' && needs.check_changes.outputs.relevant_changes == 'true' | |
| run: | | |
| git diff --name-only -- '*.snap' > snapshot-files | |
| git ls-files -o --exclude-standard -- '*.snap' >> snapshot-files | |
| if [ -s snapshot-files ]; then | |
| tr '\n' '\0' < snapshot-files > snapshot-files-0 | |
| tar --null -czf integration-snapshots-part-${{ matrix.label }}.tar.gz --files-from snapshot-files-0 | |
| echo "created=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "No snapshot files found" | |
| echo "created=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| rm -f snapshot-files snapshot-files-0 | |
| shell: bash | |
| - name: Store integration snapshots artifact | |
| if: github.event.inputs.update_snapshots == 'always' && steps.create_snapshot_archive.outputs.created == 'true' && needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: integration-snapshots-part-${{ matrix.label }} | |
| path: integration-snapshots-part-${{ matrix.label }}.tar.gz | |
| retention-days: 1 | |
| upload-integration-snapshots: | |
| name: Upload Integration Snapshots | |
| if: github.event.inputs.update_snapshots == 'always' | |
| needs: | |
| - test | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| runs-on: spiceai-dev-runners | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 | |
| with: | |
| fetch-depth: 1 | |
| - name: Download integration snapshots artifacts | |
| id: download_snapshots | |
| continue-on-error: true | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| pattern: integration-snapshots-part-* | |
| merge-multiple: true | |
| path: integration-snapshots | |
| - name: Extract snapshots | |
| if: steps.download_snapshots.outcome == 'success' | |
| run: | | |
| shopt -s nullglob | |
| for archive in integration-snapshots/*.tar.gz; do | |
| tar -xzf "$archive" -C . | |
| done | |
| shell: bash | |
| - name: Configure Git | |
| if: steps.download_snapshots.outcome == 'success' | |
| run: | | |
| git config --global user.name 'Spice Snapshot Update Bot' | |
| git config --global user.email 'spiceaibot@spice.ai' | |
| - name: Push snapshots to branch | |
| if: steps.download_snapshots.outcome == 'success' | |
| uses: ./.github/actions/push-snap-changes | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: No snapshot changes detected | |
| if: steps.download_snapshots.outcome != 'success' | |
| run: echo "No snapshot artifacts were uploaded; skipping snapshot update." | |
| test-aws-sdk: | |
| name: AWS SDK Integration Tests | |
| needs: [build, check_changes] | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| permissions: read-all | |
| runs-on: spiceai-dev-runners | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 | |
| with: | |
| fetch-depth: 1 | |
| - name: Set up Rust | |
| uses: ./.github/actions/setup-rust | |
| - name: Download AWS SDK test archive | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: integration-aws-sdk-test-archive | |
| path: ./integration_aws_sdk_test | |
| - name: Set up Nextest | |
| uses: ./.github/actions/setup-nextest | |
| - name: Login to ACR | |
| uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 | |
| if: github.repository == 'spiceai/spiceai' | |
| with: | |
| registry: spiceaitestimages.azurecr.io | |
| username: spiceai-repo-pull | |
| password: ${{ secrets.AZCR_PASSWORD }} | |
| - name: Run AWS SDK integration test | |
| env: | |
| AWS_EC2_METADATA_DISABLED: true | |
| TEST_DATABRICKS_TOKEN: ${{ secrets.NEW_DATABRICKS_TOKEN }} | |
| AWS_DATABRICKS_DELTA_ACCESS_KEY_ID: ${{ secrets.AWS_DATABRICKS_DELTA_ACCESS_KEY_ID }} | |
| AWS_DATABRICKS_DELTA_SECRET_ACCESS_KEY: ${{ secrets.AWS_DATABRICKS_DELTA_SECRET_ACCESS_KEY }} | |
| run: | | |
| INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --archive-file ./integration_aws_sdk_test/integration_aws_sdk.tar.zst | |
| test-aws-sdk-credential-bridge: | |
| name: AWS SDK Credential Bridge Integration Tests | |
| needs: [build, check_changes] | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| permissions: read-all | |
| runs-on: spiceai-dev-runners | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 | |
| with: | |
| fetch-depth: 1 | |
| - name: Set up Rust | |
| uses: ./.github/actions/setup-rust | |
| - name: Download AWS SDK credential bridge test archive | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: integration-aws-sdk-credential-bridge-test-archive | |
| path: ./integration_aws_sdk_credential_bridge_test | |
| - name: Set up Nextest | |
| uses: ./.github/actions/setup-nextest | |
| - name: Login to ACR | |
| uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 | |
| if: github.repository == 'spiceai/spiceai' | |
| with: | |
| registry: spiceaitestimages.azurecr.io | |
| username: spiceai-repo-pull | |
| password: ${{ secrets.AZCR_PASSWORD }} | |
| - name: Run AWS SDK credential bridge integration test | |
| env: | |
| AWS_EC2_METADATA_DISABLED: true | |
| AWS_S3_CLIENT_ID: ${{ secrets.AWS_S3_CLIENT_ID }} | |
| AWS_S3_IDENTITY_POOL_ID: ${{ secrets.AWS_S3_IDENTITY_POOL_ID }} | |
| AWS_S3_USERNAME: ${{ secrets.AWS_S3_USERNAME }} | |
| AWS_S3_PASSWORD: ${{ secrets.AWS_S3_PASSWORD }} | |
| AWS_COGNITO_IDP_URI: ${{ vars.AWS_COGNITO_IDP_URI }} | |
| AWS_REGION: ${{ vars.AWS_REGION }} | |
| AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }} | |
| run: | | |
| INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --archive-file ./integration_aws_sdk_credential_bridge_test/integration_aws_sdk_credential_bridge.tar.zst | |
| test-data-components: | |
| name: Data Components Integration Tests | |
| needs: [build, check_changes] | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| permissions: read-all | |
| runs-on: spiceai-dev-runners | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 | |
| with: | |
| fetch-depth: 1 | |
| - name: Set up Rust | |
| uses: ./.github/actions/setup-rust | |
| - name: Download data components test archive | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: integration-data-components-test-archive | |
| path: ./data_components_test | |
| - name: Set up Nextest | |
| uses: ./.github/actions/setup-nextest | |
| - name: Run data components integration test | |
| env: | |
| MINIO_ENDPOINT: ${{ secrets.TEST_MINIO_ENDPOINT }} | |
| MINIO_ACCESS_KEY_ID: ${{ secrets.TEST_MINIO_ACCESS_KEY }} | |
| MINIO_SECRET_ACCESS_KEY: ${{ secrets.TEST_MINIO_SECRET_KEY }} | |
| run: | | |
| INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --archive-file ./data_components_test/data_components_hadoop.tar.zst | |
| test-runtime-partition: | |
| name: Partition Table Integration Tests | |
| needs: [build, check_changes] | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| runs-on: spiceai-dev-runners | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 | |
| with: | |
| fetch-depth: 1 | |
| - name: Set up Rust | |
| uses: ./.github/actions/setup-rust | |
| - name: Download partition table test archive | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: integration-partition-table-test-archive | |
| path: ./partition_table_test | |
| - name: Set up Nextest | |
| uses: ./.github/actions/setup-nextest | |
| - name: Login to ACR | |
| uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 | |
| if: github.repository == 'spiceai/spiceai' | |
| with: | |
| registry: spiceaitestimages.azurecr.io | |
| username: spiceai-repo-pull | |
| password: ${{ secrets.AZCR_PASSWORD }} | |
| - name: Use public ECR for forks | |
| if: github.repository != 'spiceai/spiceai' | |
| run: echo "CONTAINER_REGISTRY=public.ecr.aws/docker/library/" >> $GITHUB_ENV | |
| - name: Run partition table integration test | |
| env: | |
| INSTA_UPDATE: ${{ github.event.inputs.update_snapshots }} | |
| run: | | |
| INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --archive-file ./partition_table_test/partition_table_test.tar.zst | |
| - name: Configure Git | |
| if: github.event.inputs.update_snapshots == 'always' | |
| run: | | |
| git config --global user.name 'Spice Snapshot Update Bot' | |
| git config --global user.email 'spiceaibot@spice.ai' | |
| - name: Push snapshots to branch | |
| if: github.event.inputs.update_snapshots == 'always' | |
| uses: ./.github/actions/push-snap-changes | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} |