Rc5 release notes (#10890) #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: integration tests (llms) | |
| on: | |
| push: | |
| branches: | |
| - trunk | |
| - release-* | |
| - release/* | |
| paths: | |
| - 'crates/llms/**' | |
| pull_request: | |
| branches: | |
| - trunk | |
| - release-* | |
| - release/* | |
| paths: | |
| - 'crates/llms/**' | |
| merge_group: | |
| branches: | |
| - trunk | |
| - release-* | |
| - release/* | |
| workflow_dispatch: | |
| inputs: | |
| run: | |
| description: 'Which groups of models to run' | |
| required: true | |
| default: all | |
| type: choice | |
| options: | |
| - all | |
| - hosted | |
| - self-hosted | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'trunk' && github.sha || 'any-sha' }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| CARGO_INCREMENTAL: 0 | |
| CARGO_NET_GIT_FETCH_WITH_CLI: true | |
| # CI performance optimizations | |
| CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse | |
| CARGO_NET_RETRY: 10 | |
| CARGO_HTTP_TIMEOUT: 60 | |
| jobs: | |
| setup-matrix: | |
| name: Setup strategy matrix | |
| runs-on: spiceai-dev-runners | |
| outputs: | |
| matrix: ${{ steps.setup-matrix.outputs.result }} | |
| steps: | |
| - name: Set up matrix | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| id: setup-matrix | |
| with: | |
| script: | | |
| const matrix = [ | |
| { | |
| name: 'self-hosted', | |
| needs_env: false, | |
| runner: 'spiceai-macos', | |
| model_allowlist: 'local_phi3,hf_phi3' | |
| }, | |
| { | |
| name: 'hosted', | |
| needs_env: true, | |
| runner: 'spiceai-macos', | |
| model_allowlist: 'anthropic,bedrock,openai,xai' | |
| } | |
| ]; | |
| if (context.eventName === 'workflow_dispatch') { | |
| const name = context.payload.inputs.run; | |
| if (name === 'all') { | |
| return matrix; | |
| } | |
| return matrix.filter(m => m.name === name); | |
| } | |
| // On pull_request, only run tests that don't require secrets | |
| if (context.eventName === 'pull_request') { | |
| return matrix.filter(m => !m.needs_env); | |
| } | |
| return matrix; | |
| build: | |
| name: Build Test Binary | |
| runs-on: spiceai-macos | |
| env: | |
| HAS_SPICEIO_SECRET: ${{ secrets.UNAS_SMB_PASS != '' }} | |
| TEST_BINARY_OBJECT_KEY: test_binaries/${{ github.run_id }}/llms_integration_test | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| fetch-depth: 1 # Shallow clone for faster checkout | |
| - name: Set up Rust | |
| uses: ./.github/actions/setup-rust | |
| - name: Build LLMs integration test binary | |
| env: | |
| # Metal runners do not have `xcrun metal` | |
| # See `https://github.com/EricLBuehler/mistral.rs/pull/1311` | |
| MISTRALRS_METAL_PRECOMPILE: 0 | |
| run: | | |
| TEST_BINARY_PATH=$(cargo test -p llms --test integration --features metal --no-run --message-format=json | jq -r 'select(.reason == "compiler-artifact" and (.target.kind | contains(["test"])) and .executable != null) | .executable') | |
| cp $TEST_BINARY_PATH ./llms_integration_test | |
| - name: Set up spiceio | |
| if: runner.os == 'macOS' && env.HAS_SPICEIO_SECRET == 'true' | |
| id: setup-spiceio | |
| uses: spiceai/spiceio/.github/actions/setup@e31c63cbe88a6e284fc187875261106a05034775 # v0.5.4 | |
| with: | |
| smb-url: smb://runner@192.168.3.148/ai_platform_dev | |
| smb-pass: ${{ secrets.UNAS_SMB_PASS }} | |
| token: ${{ github.token }} | |
| bucket: build | |
| region: us-west-1 | |
| - name: Upload test binary to spiceio | |
| if: env.HAS_SPICEIO_SECRET == 'true' | |
| env: | |
| SPICEIO_ENDPOINT: ${{ steps.setup-spiceio.outputs.endpoint }} | |
| run: | | |
| set -euo pipefail | |
| : "${SPICEIO_ENDPOINT:?spiceio endpoint is required to upload the LLM integration test binary}" | |
| test_binary_url="${SPICEIO_ENDPOINT%/}/build/${TEST_BINARY_OBJECT_KEY}" | |
| curl --fail --silent --show-error --upload-file ./llms_integration_test "$test_binary_url" | |
| - name: Upload test binary | |
| if: env.HAS_SPICEIO_SECRET != 'true' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: llms-integration-test-binary | |
| path: ./llms_integration_test | |
| retention-days: 3 | |
| compression-level: 1 | |
| - name: Kill spiceio | |
| if: always() && runner.os == 'macOS' && steps.setup-spiceio.outputs.pid != '' | |
| run: kill ${{ steps.setup-spiceio.outputs.pid }} || true | |
| test: | |
| runs-on: ${{ matrix.target.runner }} | |
| needs: [build, setup-matrix] | |
| env: | |
| HAS_SPICEIO_SECRET: ${{ secrets.UNAS_SMB_PASS != '' }} | |
| TEST_BINARY_OBJECT_KEY: test_binaries/${{ github.run_id }}/llms_integration_test | |
| strategy: | |
| matrix: | |
| target: ${{ fromJson(needs.setup-matrix.outputs.matrix) }} | |
| permissions: read-all | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| - name: Set up spiceio | |
| if: runner.os == 'macOS' && env.HAS_SPICEIO_SECRET == 'true' | |
| id: setup-spiceio | |
| uses: spiceai/spiceio/.github/actions/setup@e31c63cbe88a6e284fc187875261106a05034775 # v0.5.4 | |
| with: | |
| smb-url: smb://runner@192.168.3.148/ai_platform_dev | |
| smb-pass: ${{ secrets.UNAS_SMB_PASS }} | |
| token: ${{ github.token }} | |
| bucket: build | |
| region: us-west-1 | |
| - name: Download test binary from spiceio | |
| if: env.HAS_SPICEIO_SECRET == 'true' | |
| env: | |
| SPICEIO_ENDPOINT: ${{ steps.setup-spiceio.outputs.endpoint }} | |
| run: | | |
| set -euo pipefail | |
| : "${SPICEIO_ENDPOINT:?spiceio endpoint is required to download the LLM integration test binary}" | |
| mkdir -p ./integration_test | |
| test_binary_url="${SPICEIO_ENDPOINT%/}/build/${TEST_BINARY_OBJECT_KEY}" | |
| curl --fail --silent --show-error --output ./integration_test/llms_integration_test "$test_binary_url" | |
| - name: Download test binary | |
| if: env.HAS_SPICEIO_SECRET != 'true' | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: llms-integration-test-binary | |
| path: ./integration_test | |
| - name: Mark test binary as executable | |
| run: chmod +x ./integration_test/llms_integration_test | |
| - name: Run integration test | |
| env: | |
| MODEL_ALLOWLIST: ${{ matrix.target.model_allowlist }} | |
| SPICE_OPENAI_API_KEY: ${{ secrets.SPICE_SECRET_OPENAI_API_KEY }} | |
| SPICE_ANTHROPIC_API_KEY: ${{ secrets.SPICE_SECRET_ANTHROPIC_API_KEY }} | |
| SPICE_XAI_API_KEY: ${{ secrets.SPICE_SECRET_XAI_API_KEY }} | |
| SPICE_BEDROCK_ACCESS_KEY: ${{ secrets.AWS_BEDROCK_KEY }} | |
| SPICE_BEDROCK_SECRET_KEY: ${{ secrets.AWS_BEDROCK_SECRET }} | |
| run: | | |
| if [ "${{ matrix.target.needs_env }}" == "true" ]; then | |
| if [ -z "$SPICE_OPENAI_API_KEY" ]; then | |
| echo "Error: 'SPICE_OPENAI_API_KEY' is not defined." | |
| exit 1 | |
| fi | |
| if [ -z "$SPICE_ANTHROPIC_API_KEY" ]; then | |
| echo "Error: 'SPICE_ANTHROPIC_API_KEY' is not defined." | |
| exit 1 | |
| fi | |
| if [ -z "$SPICE_XAI_API_KEY" ]; then | |
| echo "Error: 'SPICE_XAI_API_KEY' is not defined." | |
| exit 1 | |
| fi | |
| export SPICE_BEDROCK_REGION="us-east-1" | |
| if [ -z "$SPICE_BEDROCK_ACCESS_KEY" ]; then | |
| echo "Error: 'SPICE_BEDROCK_ACCESS_KEY' is not defined." | |
| exit 1 | |
| fi | |
| if [ -z "$SPICE_BEDROCK_SECRET_KEY" ]; then | |
| echo "Error: 'SPICE_BEDROCK_SECRET_KEY' is not defined." | |
| exit 1 | |
| fi | |
| fi | |
| # `--test-threads` to reduce possible rate limiting/ connection issues for hosted models. | |
| INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" ./integration_test/llms_integration_test --nocapture -- llms::tests --test-threads=2 | |
| - name: Kill spiceio | |
| if: always() && runner.os == 'macOS' && steps.setup-spiceio.outputs.pid != '' | |
| run: kill ${{ steps.setup-spiceio.outputs.pid }} || true |