Skip to content

fix: Provide a more informative 404 on async API when result has no c… #1880

fix: Provide a more informative 404 on async API when result has no c…

fix: Provide a more informative 404 on async API when result has no c… #1880

---
name: integration tests (models)
on:
push:
branches:
- trunk
- release/*
pull_request_target:
branches:
- trunk
- release-*
- release/*
- feature-*
paths-ignore:
- '**/*.md'
- 'docs/**'
- 'README.md'
- 'Makefile'
- 'CONTRIBUTING.md'
- 'SECURITY.md'
- 'LICENSE'
- '.github/**'
- 'version.txt'
- '.schema/**'
- '.vscode/**'
- 'deploy/**'
- 'install/**'
- 'media/**'
- 'monitoring/**'
- 'acknowledgements.md'
- 'Dockerfile*'
- 'bin/spice/**'
schedule:
- cron: '0 0 * * *' # Daily at midnight UTC
workflow_dispatch:
inputs:
run_all_tests:
description: 'Run all tests'
type: boolean
required: false
default: false
update_snapshots:
description: 'Update the snapshots?'
required: false
default: 'no'
type: choice
options:
- 'always'
- 'no'
concurrency:
# Allow only one workflow per any non-trunk branch.
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_name == 'trunk' && github.sha || 'any-sha' }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: 0
CARGO_NET_GIT_FETCH_WITH_CLI: true
# CI performance optimizations
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
CARGO_NET_RETRY: 10
CARGO_HTTP_TIMEOUT: 60
jobs:
build:
name: Build Test Binary (macOS)
runs-on: 'spiceai-macos'
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 1 # Shallow clone for faster checkout
- name: Set up Rust
uses: ./.github/actions/setup-rust
- name: Set up sccache
uses: ./.github/actions/setup-sccache
with:
minio_endpoint: ${{ secrets.TEST_MINIO_ENDPOINT }}
- name: Set up Nextest
uses: ./.github/actions/setup-nextest
# Build the test binary without running tests
- name: Build AI integration test binary
env:
# Metal runners do not have `xcrun metal`
# See `https://github.com/EricLBuehler/mistral.rs/pull/1311`
MISTRALRS_METAL_PRECOMPILE: 0
RUSTC_WRAPPER: sccache
AWS_ACCESS_KEY_ID: ${{ secrets.TEST_MINIO_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_MINIO_SECRET_KEY }}
run: |
FEATURES="flightsql,models,metal,s3_vectors,kafka"
if [[ "${{ github.event.inputs.run_all_tests }}" == "true" ]] || [[ "${{ github.event_name }}" == "schedule" ]]; then
FEATURES="${FEATURES},bedrock,extended_tests"
echo "Including extended_tests"
else
echo "Excluding extended_tests"
fi
cargo nextest archive -p runtime --test integration_models --features ${FEATURES} --archive-file integration.tar.zst
# Upload the test archive as an artifact
- name: Upload test archive
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v5
with:
name: ai-integration-test-archive
path: ./integration.tar.zst
retention-days: 1
test-openai:
name: Test OpenAI Models
needs: build
runs-on: 'spiceai-macos'
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Rust
uses: ./.github/actions/setup-rust
- name: Set up Nextest
uses: ./.github/actions/setup-nextest
- name: Download test archive
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v6
with:
name: ai-integration-test-archive
path: ./integration_test
- name: Set up Open.ai API Key
run: |
echo 'SPICE_OPENAI_API_KEY="${{ secrets.SPICE_SECRET_OPENAI_API_KEY }}"' > .env
- name: Run integration test
env:
INSTA_UPDATE: ${{ github.event_name == 'schedule' && 'always' || github.event.inputs.update_snapshots }}
SPICE_SECRET_OPENAI_API_KEY: ${{ secrets.SPICE_SECRET_OPENAI_API_KEY }}
run: |
if [ -z "$SPICE_SECRET_OPENAI_API_KEY" ] ; then
echo "Error: OpenAI API key is not defined."
exit 1
fi
INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --archive-file ./integration_test/integration.tar.zst -- openai_test
- name: Run Beta functionality embeddings test
env:
INSTA_UPDATE: ${{ github.event_name == 'schedule' && 'always' || github.event.inputs.update_snapshots }}
SPICE_SECRET_OPENAI_API_KEY: ${{ secrets.SPICE_SECRET_OPENAI_API_KEY }}
run: |
if [ -z "$SPICE_SECRET_OPENAI_API_KEY" ] ; then
echo "Error: OpenAI API key is not defined."
exit 1
fi
INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --archive-file ./integration_test/integration.tar.zst -- openai_embeddings_beta_requirements
- name: Push snapshots to branch
if: github.event.inputs.update_snapshots == 'always' || github.event_name == 'schedule'
uses: ./.github/actions/push-snap-changes
with:
token: ${{ secrets.GITHUB_TOKEN }}
test-ai-udf:
name: Test AI UDFs
needs: build
runs-on: 'spiceai-macos'
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Rust
uses: ./.github/actions/setup-rust
- name: Set up Nextest
uses: ./.github/actions/setup-nextest
- name: Download test archive
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v6
with:
name: ai-integration-test-archive
path: ./integration_test
- name: Set up API Keys
run: |
echo 'SPICE_OPENAI_API_KEY="${{ secrets.SPICE_SECRET_OPENAI_API_KEY }}"' > .env
echo 'SPICE_ANTHROPIC_API_KEY="${{ secrets.SPICE_SECRET_ANTHROPIC_API_KEY }}"' >> .env
echo 'SPICE_XAI_API_KEY="${{ secrets.SPICE_SECRET_XAI_API_KEY }}"' >> .env
- name: Run AI UDF basic test
env:
INSTA_UPDATE: ${{ github.event_name == 'schedule' && 'always' || github.event.inputs.update_snapshots }}
SPICE_SECRET_OPENAI_API_KEY: ${{ secrets.SPICE_SECRET_OPENAI_API_KEY }}
SPICE_SECRET_ANTHROPIC_API_KEY: ${{ secrets.SPICE_SECRET_ANTHROPIC_API_KEY }}
run: |
if [ -z "$SPICE_SECRET_OPENAI_API_KEY" ] ; then
echo "Error: OpenAI API key is not defined."
exit 1
fi
if [ -z "$SPICE_SECRET_ANTHROPIC_API_KEY" ] ; then
echo "Error: Anthropic API key is not defined."
exit 1
fi
INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --archive-file ./integration_test/integration.tar.zst -- test_ai_udf_basic
- name: Run AI UDF with dataset test
env:
INSTA_UPDATE: ${{ github.event_name == 'schedule' && 'always' || github.event.inputs.update_snapshots }}
SPICE_SECRET_OPENAI_API_KEY: ${{ secrets.SPICE_SECRET_OPENAI_API_KEY }}
SPICE_SECRET_ANTHROPIC_API_KEY: ${{ secrets.SPICE_SECRET_ANTHROPIC_API_KEY }}
SPICE_SECRET_XAI_API_KEY: ${{ secrets.SPICE_SECRET_XAI_API_KEY }}
run: |
if [ -z "$SPICE_SECRET_OPENAI_API_KEY" ] ; then
echo "Error: OpenAI API key is not defined."
exit 1
fi
if [ -z "$SPICE_SECRET_ANTHROPIC_API_KEY" ] ; then
echo "Error: Anthropic API key is not defined."
exit 1
fi
if [ -z "$SPICE_SECRET_XAI_API_KEY" ] ; then
echo "Error: xAI API key is not defined."
exit 1
fi
INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --archive-file ./integration_test/integration.tar.zst -- test_ai_udf_with_dataset
- name: Run AI UDF LEFT truncate test
env:
INSTA_UPDATE: ${{ github.event_name == 'schedule' && 'always' || github.event.inputs.update_snapshots }}
SPICE_SECRET_OPENAI_API_KEY: ${{ secrets.SPICE_SECRET_OPENAI_API_KEY }}
SPICE_SECRET_ANTHROPIC_API_KEY: ${{ secrets.SPICE_SECRET_ANTHROPIC_API_KEY }}
SPICE_SECRET_XAI_API_KEY: ${{ secrets.SPICE_SECRET_XAI_API_KEY }}
run: |
if [ -z "$SPICE_SECRET_OPENAI_API_KEY" ] ; then
echo "Error: OpenAI API key is not defined."
exit 1
fi
if [ -z "$SPICE_SECRET_ANTHROPIC_API_KEY" ] ; then
echo "Error: Anthropic API key is not defined."
exit 1
fi
if [ -z "$SPICE_SECRET_XAI_API_KEY" ] ; then
echo "Error: xAI API key is not defined."
exit 1
fi
INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --archive-file ./integration_test/integration.tar.zst -- test_ai_udf_left_truncate
- name: Push snapshots to branch
if: github.event.inputs.update_snapshots == 'always' || github.event_name == 'schedule'
uses: ./.github/actions/push-snap-changes
with:
token: ${{ secrets.GITHUB_TOKEN }}
test-hf:
name: Test Hugging Face Models
needs: build
if: ${{ github.event.inputs.run_all_tests == 'true' || github.event_name == 'schedule' }}
runs-on: 'spiceai-macos'
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Rust
uses: ./.github/actions/setup-rust
- name: Set up Nextest
uses: ./.github/actions/setup-nextest
- name: Download test archive
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v6
with:
name: ai-integration-test-archive
path: ./integration_test
- name: Run integration test
env:
INSTA_UPDATE: ${{ github.event_name == 'schedule' && 'always' || github.event.inputs.update_snapshots }}
SPICE_HF_TOKEN: ${{ secrets.SPICE_SECRET_HF_TOKEN }}
run: |
if [ -z "$SPICE_HF_TOKEN" ] ; then
echo "Error: Hugging Face API Token is not defined."
exit 1
fi
INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --archive-file ./integration_test/integration.tar.zst -- huggingface_test
- name: Run Beta functionality embedding tests
env:
INSTA_UPDATE: ${{ github.event_name == 'schedule' && 'always' || github.event.inputs.update_snapshots }}
SPICE_HF_TOKEN: ${{ secrets.SPICE_SECRET_HF_TOKEN }}
run: |
if [ -z "$SPICE_HF_TOKEN" ] ; then
echo "Error: Hugging Face API Token is not defined."
exit 1
fi
INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --archive-file ./integration_test/integration.tar.zst -- hf_embeddings_beta_requirements
- name: Push snapshots to branch
if: github.event.inputs.update_snapshots == 'always' || github.event_name == 'schedule'
uses: ./.github/actions/push-snap-changes
with:
token: ${{ secrets.GITHUB_TOKEN }}
test-local:
name: Test Local Models
needs: build
runs-on: 'spiceai-macos'
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Rust
uses: ./.github/actions/setup-rust
- name: Set up Nextest
uses: ./.github/actions/setup-nextest
- name: Download test archive
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v6
with:
name: ai-integration-test-archive
path: ./integration_test
- name: Run Beta functionality embedding tests
env:
INSTA_UPDATE: ${{ github.event_name == 'schedule' && 'always' || github.event.inputs.update_snapshots }}
run: |
INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --archive-file ./integration_test/integration.tar.zst -- local_embeddings_beta_requirements
- name: Run AI UDF with local model test
env:
INSTA_UPDATE: ${{ github.event_name == 'schedule' && 'always' || github.event.inputs.update_snapshots }}
SPICE_HF_TOKEN: ${{ secrets.SPICE_SECRET_HF_TOKEN }}
run: |
if [ -z "$SPICE_HF_TOKEN" ] ; then
echo "Error: Hugging Face API Token is not defined."
exit 1
fi
INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --archive-file ./integration_test/integration.tar.zst -- test_ai_udf_with_local_model
- name: Push snapshots to branch
if: github.event.inputs.update_snapshots == 'always' || github.event_name == 'schedule'
uses: ./.github/actions/push-snap-changes
with:
token: ${{ secrets.GITHUB_TOKEN }}
test-bedrock:
name: Test Bedrock Models
needs: build
if: ${{ github.event.inputs.run_all_tests == 'true' || github.event_name == 'schedule' }}
runs-on: 'spiceai-macos'
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Rust
uses: ./.github/actions/setup-rust
- name: Set up Nextest
uses: ./.github/actions/setup-nextest
- name: Download test archive
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v6
with:
name: ai-integration-test-archive
path: ./integration_test
- name: Run Beta functionality embedding tests
env:
INSTA_UPDATE: ${{ github.event_name == 'schedule' && 'always' || github.event.inputs.update_snapshots }}
AWS_BEDROCK_KEY: ${{ secrets.AWS_BEDROCK_KEY }}
AWS_BEDROCK_SECRET: ${{ secrets.AWS_BEDROCK_SECRET }}
run: |
INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --archive-file ./integration_test/integration.tar.zst -- bedrock_test
- name: Push snapshots to branch
if: github.event.inputs.update_snapshots == 'always' || github.event_name == 'schedule'
uses: ./.github/actions/push-snap-changes
with:
token: ${{ secrets.GITHUB_TOKEN }}
test-workers:
name: Test Workers
needs: build
runs-on: 'spiceai-macos'
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Rust
uses: ./.github/actions/setup-rust
- name: Set up Nextest
uses: ./.github/actions/setup-nextest
- name: Download test archive
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v6
with:
name: ai-integration-test-archive
path: ./integration_test
- name: Set up Open.ai API Key
run: |
echo 'SPICE_OPENAI_API_KEY="${{ secrets.SPICE_SECRET_OPENAI_API_KEY }}"' > .env
- name: Run integration test
env:
INSTA_UPDATE: ${{ github.event_name == 'schedule' && 'always' || github.event.inputs.update_snapshots }}
SPICE_SECRET_OPENAI_API_KEY: ${{ secrets.SPICE_SECRET_OPENAI_API_KEY }}
run: |
if [ -z "$SPICE_SECRET_OPENAI_API_KEY" ] ; then
echo "Error: OpenAI API key is not defined."
exit 1
fi
INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --archive-file ./integration_test/integration.tar.zst -- workers
- name: Push snapshots to branch
if: github.event.inputs.update_snapshots == 'always' || github.event_name == 'schedule'
uses: ./.github/actions/push-snap-changes
with:
token: ${{ secrets.GITHUB_TOKEN }}
build-extended:
name: Build Test Binary (Extended)
runs-on: 'spiceai-macos'
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 1 # Shallow clone for faster checkout
- name: Set up Rust
uses: ./.github/actions/setup-rust
- name: Set up sccache
uses: ./.github/actions/setup-sccache
with:
minio_endpoint: ${{ secrets.TEST_MINIO_ENDPOINT }}
- name: Set up Nextest
uses: ./.github/actions/setup-nextest
# Build the test binary without running tests
- name: Build AI integration test binary
env:
RUSTC_WRAPPER: sccache
AWS_ACCESS_KEY_ID: ${{ secrets.TEST_MINIO_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_MINIO_SECRET_KEY }}
run: |
FEATURES="flightsql,models,s3_vectors,kafka"
if [[ "${{ github.event.inputs.run_all_tests }}" == "true" ]] || [[ "${{ github.event_name }}" == "schedule" ]]; then
FEATURES="${FEATURES},bedrock,extended_tests"
echo "Including extended_tests"
else
echo "Excluding extended_tests"
fi
cargo nextest archive -p runtime --test integration_models --features ${FEATURES} --archive-file integration-extended.tar.zst
# Upload the test archive as an artifact
- name: Upload test archive
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v5
with:
name: ai-integration-test-archive-extended
path: ./integration-extended.tar.zst
retention-days: 1
test-search:
name: Test Search
needs: build
runs-on: 'spiceai-macos'
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Rust
uses: ./.github/actions/setup-rust
- name: Set up Nextest
uses: ./.github/actions/setup-nextest
- name: Download test archive
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v6
with:
name: ai-integration-test-archive
path: ./integration_test
- name: Set up Open.ai API Key
run: |
echo 'SPICE_OPENAI_API_KEY="${{ secrets.SPICE_SECRET_OPENAI_API_KEY }}"' > .env
- name: Run integration test
env:
INSTA_UPDATE: ${{ github.event_name == 'schedule' && 'always' || github.event.inputs.update_snapshots }}
SPICE_SECRET_OPENAI_API_KEY: ${{ secrets.SPICE_SECRET_OPENAI_API_KEY }}
AWS_S3_VECTORS_KEY: ${{ secrets.AWS_S3_VECTORS_KEY }}
AWS_S3_VECTORS_SECRET: ${{ secrets.AWS_S3_VECTORS_SECRET }}
run: |
if [ -z "$SPICE_SECRET_OPENAI_API_KEY" ] ; then
echo "Error: OpenAI API key is not defined."
exit 1
fi
INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" cargo nextest run --workspace-remap "${PWD}" --archive-file ./integration_test/integration.tar.zst -- search
- name: Push snapshots to branch
if: github.event.inputs.update_snapshots == 'always' || github.event_name == 'schedule'
uses: ./.github/actions/push-snap-changes
with:
token: ${{ secrets.GITHUB_TOKEN }}