integration_llms.yml

---
name: integration tests (llms)

on:
  push:
    branches:
      - trunk
      - release-*
      - release/*
    paths:
      - 'crates/llms/**'
  pull_request:
    branches:
      - trunk
      - release-*
      - release/*
    paths:
      - 'crates/llms/**'
  merge_group:
    branches:
      - trunk
      - release-*
      - release/*
  workflow_dispatch:
    inputs:
      run:
        description: 'Which groups of models to run'
        required: true
        default: all
        type: choice
        options:
          - all
          - hosted
          - self-hosted

concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'trunk' && github.sha || 'any-sha' }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always
  CARGO_INCREMENTAL: 0
  CARGO_NET_GIT_FETCH_WITH_CLI: true
  # CI performance optimizations
  CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
  CARGO_NET_RETRY: 10
  CARGO_HTTP_TIMEOUT: 60

jobs:
  setup-matrix:
    name: Setup strategy matrix
    runs-on: spiceai-dev-runners
    outputs:
      matrix: ${{ steps.setup-matrix.outputs.result }}

    steps:
      - name: Set up matrix
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        id: setup-matrix
        with:
          script: |
            const matrix = [
              {
                name: 'self-hosted',
                needs_env: false,
                runner: 'spiceai-macos',
                model_allowlist: 'local_phi3,hf_phi3'
              },
              {
                name: 'hosted',
                needs_env: true,
                runner: 'spiceai-macos',
                model_allowlist: 'anthropic,bedrock,openai,xai'
              }
            ];

            if (context.eventName === 'workflow_dispatch') {
              const name = context.payload.inputs.run;

              if (name === 'all') {
                return matrix;
              }

              return matrix.filter(m => m.name === name);
            }

            // On pull_request, only run tests that don't require secrets
            if (context.eventName === 'pull_request') {
              return matrix.filter(m => !m.needs_env);
            }

            return matrix;

  build:
    name: Build Test Binary
    runs-on: spiceai-macos
    steps:
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
        with:
          fetch-depth: 1 # Shallow clone for faster checkout

      - name: Set up Rust
        uses: ./.github/actions/setup-rust

      - name: Build LLMs integration test binary
        env:
          # Metal runners do not have `xcrun metal`
          # See `https://github.com/EricLBuehler/mistral.rs/pull/1311`
          MISTRALRS_METAL_PRECOMPILE: 0
        run: |
          TEST_BINARY_PATH=$(cargo test -p llms --test integration --features metal --no-run --message-format=json | jq -r 'select(.reason == "compiler-artifact" and (.target.kind | contains(["test"])) and .executable != null) | .executable')
          cp $TEST_BINARY_PATH ./llms_integration_test

      - name: Upload test binary
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: llms-integration-test-binary
          path: ./llms_integration_test
          retention-days: 1

  test:
    runs-on: ${{ matrix.target.runner }}
    needs: [build, setup-matrix]
    strategy:
      matrix:
        target: ${{ fromJson(needs.setup-matrix.outputs.matrix) }}
    permissions: read-all

    steps:
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
      - name: Download test binary
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: llms-integration-test-binary
          path: ./integration_test

      - name: Mark test binary as executable
        run: chmod +x ./integration_test/llms_integration_test

      - name: Run integration test
        env:
          MODEL_ALLOWLIST: ${{ matrix.target.model_allowlist }}
          SPICE_OPENAI_API_KEY: ${{ secrets.SPICE_SECRET_OPENAI_API_KEY }}
          SPICE_ANTHROPIC_API_KEY: ${{ secrets.SPICE_SECRET_ANTHROPIC_API_KEY }}
          SPICE_XAI_API_KEY: ${{ secrets.SPICE_SECRET_XAI_API_KEY }}
          SPICE_BEDROCK_ACCESS_KEY: ${{ secrets.AWS_BEDROCK_KEY }}
          SPICE_BEDROCK_SECRET_KEY: ${{ secrets.AWS_BEDROCK_SECRET }}
        run: |
          if [ "${{ matrix.target.needs_env }}" == "true" ]; then
            if [ -z "$SPICE_OPENAI_API_KEY" ]; then
              echo "Error: 'SPICE_OPENAI_API_KEY' is not defined."
              exit 1
            fi
            if [ -z "$SPICE_ANTHROPIC_API_KEY" ]; then
              echo "Error: 'SPICE_ANTHROPIC_API_KEY' is not defined."
              exit 1
            fi
            if [ -z "$SPICE_XAI_API_KEY" ]; then
              echo "Error: 'SPICE_XAI_API_KEY' is not defined."
              exit 1
            fi
            export SPICE_BEDROCK_REGION="us-east-1"
            if [ -z "$SPICE_BEDROCK_ACCESS_KEY" ]; then
              echo "Error: 'SPICE_BEDROCK_ACCESS_KEY' is not defined."
              exit 1
            fi
            if [ -z "$SPICE_BEDROCK_SECRET_KEY" ]; then
              echo "Error: 'SPICE_BEDROCK_SECRET_KEY' is not defined."
              exit 1
            fi
          fi
          # `--test-threads` to reduce possible rate limiting/ connection issues for hosted models.
          INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" ./integration_test/llms_integration_test --nocapture -- llms::tests --test-threads=2