v2.0.0-rc.3 preparation (#10382) #102
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: integration tests (llms) | |
| on: | |
| push: | |
| branches: | |
| - trunk | |
| - release-* | |
| - release/* | |
| paths: | |
| - 'crates/llms/**' | |
| pull_request: | |
| branches: | |
| - trunk | |
| - release-* | |
| - release/* | |
| paths: | |
| - 'crates/llms/**' | |
| merge_group: | |
| branches: | |
| - trunk | |
| - release-* | |
| - release/* | |
| workflow_dispatch: | |
| inputs: | |
| run: | |
| description: 'Which groups of models to run' | |
| required: true | |
| default: all | |
| type: choice | |
| options: | |
| - all | |
| - hosted | |
| - self-hosted | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'trunk' && github.sha || 'any-sha' }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| CARGO_INCREMENTAL: 0 | |
| CARGO_NET_GIT_FETCH_WITH_CLI: true | |
| # CI performance optimizations | |
| CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse | |
| CARGO_NET_RETRY: 10 | |
| CARGO_HTTP_TIMEOUT: 60 | |
| jobs: | |
| setup-matrix: | |
| name: Setup strategy matrix | |
| runs-on: spiceai-dev-runners | |
| outputs: | |
| matrix: ${{ steps.setup-matrix.outputs.result }} | |
| steps: | |
| - name: Set up matrix | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| id: setup-matrix | |
| with: | |
| script: | | |
| const matrix = [ | |
| { | |
| name: 'self-hosted', | |
| needs_env: false, | |
| runner: 'spiceai-macos', | |
| model_allowlist: 'local_phi3,hf_phi3' | |
| }, | |
| { | |
| name: 'hosted', | |
| needs_env: true, | |
| runner: 'spiceai-macos', | |
| model_allowlist: 'anthropic,bedrock,openai,xai' | |
| } | |
| ]; | |
| if (context.eventName === 'workflow_dispatch') { | |
| const name = context.payload.inputs.run; | |
| if (name === 'all') { | |
| return matrix; | |
| } | |
| return matrix.filter(m => m.name === name); | |
| } | |
| // On pull_request, only run tests that don't require secrets | |
| if (context.eventName === 'pull_request') { | |
| return matrix.filter(m => !m.needs_env); | |
| } | |
| return matrix; | |
| build: | |
| name: Build Test Binary | |
| runs-on: spiceai-macos | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 | |
| with: | |
| fetch-depth: 1 # Shallow clone for faster checkout | |
| - name: Set up Rust | |
| uses: ./.github/actions/setup-rust | |
| - name: Build LLMs integration test binary | |
| env: | |
| # Metal runners do not have `xcrun metal` | |
| # See `https://github.com/EricLBuehler/mistral.rs/pull/1311` | |
| MISTRALRS_METAL_PRECOMPILE: 0 | |
| run: | | |
| TEST_BINARY_PATH=$(cargo test -p llms --test integration --features metal --no-run --message-format=json | jq -r 'select(.reason == "compiler-artifact" and (.target.kind | contains(["test"])) and .executable != null) | .executable') | |
| cp $TEST_BINARY_PATH ./llms_integration_test | |
| - name: Upload test binary | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: llms-integration-test-binary | |
| path: ./llms_integration_test | |
| retention-days: 3 | |
| # Test binary is already a compact native binary; skip artifact zip compression. | |
| compression-level: 0 | |
| test: | |
| runs-on: ${{ matrix.target.runner }} | |
| needs: [build, setup-matrix] | |
| strategy: | |
| matrix: | |
| target: ${{ fromJson(needs.setup-matrix.outputs.matrix) }} | |
| permissions: read-all | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 | |
| - name: Download test binary | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: llms-integration-test-binary | |
| path: ./integration_test | |
| - name: Mark test binary as executable | |
| run: chmod +x ./integration_test/llms_integration_test | |
| - name: Run integration test | |
| env: | |
| MODEL_ALLOWLIST: ${{ matrix.target.model_allowlist }} | |
| SPICE_OPENAI_API_KEY: ${{ secrets.SPICE_SECRET_OPENAI_API_KEY }} | |
| SPICE_ANTHROPIC_API_KEY: ${{ secrets.SPICE_SECRET_ANTHROPIC_API_KEY }} | |
| SPICE_XAI_API_KEY: ${{ secrets.SPICE_SECRET_XAI_API_KEY }} | |
| SPICE_BEDROCK_ACCESS_KEY: ${{ secrets.AWS_BEDROCK_KEY }} | |
| SPICE_BEDROCK_SECRET_KEY: ${{ secrets.AWS_BEDROCK_SECRET }} | |
| run: | | |
| if [ "${{ matrix.target.needs_env }}" == "true" ]; then | |
| if [ -z "$SPICE_OPENAI_API_KEY" ]; then | |
| echo "Error: 'SPICE_OPENAI_API_KEY' is not defined." | |
| exit 1 | |
| fi | |
| if [ -z "$SPICE_ANTHROPIC_API_KEY" ]; then | |
| echo "Error: 'SPICE_ANTHROPIC_API_KEY' is not defined." | |
| exit 1 | |
| fi | |
| if [ -z "$SPICE_XAI_API_KEY" ]; then | |
| echo "Error: 'SPICE_XAI_API_KEY' is not defined." | |
| exit 1 | |
| fi | |
| export SPICE_BEDROCK_REGION="us-east-1" | |
| if [ -z "$SPICE_BEDROCK_ACCESS_KEY" ]; then | |
| echo "Error: 'SPICE_BEDROCK_ACCESS_KEY' is not defined." | |
| exit 1 | |
| fi | |
| if [ -z "$SPICE_BEDROCK_SECRET_KEY" ]; then | |
| echo "Error: 'SPICE_BEDROCK_SECRET_KEY' is not defined." | |
| exit 1 | |
| fi | |
| fi | |
| # `--test-threads` to reduce possible rate limiting/ connection issues for hosted models. | |
| INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" ./integration_test/llms_integration_test --nocapture -- llms::tests --test-threads=2 |