Run the integration test suite from tests/integration in replay mode #10
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Tests (Replay) | |
| run-name: Run the integration test suite from tests/integration in replay mode | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| types: [opened, synchronize, reopened] | |
| paths: | |
| - 'llama_stack/**' | |
| - '!llama_stack/ui/**' | |
| - 'tests/**' | |
| - 'uv.lock' | |
| - 'pyproject.toml' | |
| - '.github/workflows/integration-tests.yml' # This workflow | |
| - '.github/actions/setup-ollama/action.yml' | |
| - '.github/actions/setup-test-environment/action.yml' | |
| - '.github/actions/run-and-record-tests/action.yml' | |
| schedule: | |
| # If changing the cron schedule, update the provider in the test-matrix job | |
| - cron: '0 0 * * *' # (test latest client) Daily at 12 AM UTC | |
| - cron: '1 0 * * 0' # (test vllm) Weekly on Sunday at 1 AM UTC | |
| workflow_dispatch: | |
| inputs: | |
| test-all-client-versions: | |
| description: 'Test against both the latest and published versions' | |
| type: boolean | |
| default: false | |
| test-provider: | |
| description: 'Test against a specific provider' | |
| type: string | |
| default: 'ollama' | |
| test-subdirs: | |
| description: 'Comma-separated list of test subdirectories to run' | |
| type: string | |
| default: '' | |
| test-pattern: | |
| description: 'Regex pattern to pass to pytest -k' | |
| type: string | |
| default: '' | |
| concurrency: | |
| # Skip concurrency for pushes to main - each commit should be tested independently | |
| group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| run-replay-mode-tests: | |
| runs-on: ubuntu-latest | |
| name: ${{ format('Integration Tests ({0}, {1}, {2}, client={3}, vision={4})', matrix.client-type, matrix.provider, matrix.python-version, matrix.client-version, matrix.run-vision-tests) }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| client-type: [library, server] | |
| # Use vllm on weekly schedule, otherwise use test-provider input (defaults to ollama) | |
| provider: ${{ (github.event.schedule == '1 0 * * 0') && fromJSON('["vllm"]') || fromJSON(format('["{0}"]', github.event.inputs.test-provider || 'ollama')) }} | |
| # Use Python 3.13 only on nightly schedule (daily latest client test), otherwise use 3.12 | |
| python-version: ${{ github.event.schedule == '0 0 * * *' && fromJSON('["3.12", "3.13"]') || fromJSON('["3.12"]') }} | |
| client-version: ${{ (github.event.schedule == '0 0 * * *' || github.event.inputs.test-all-client-versions == 'true') && fromJSON('["published", "latest"]') || fromJSON('["latest"]') }} | |
| run-vision-tests: [true, false] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| - name: Setup test environment | |
| uses: ./.github/actions/setup-test-environment | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| client-version: ${{ matrix.client-version }} | |
| provider: ${{ matrix.provider }} | |
| run-vision-tests: ${{ matrix.run-vision-tests }} | |
| inference-mode: 'replay' | |
| - name: Run tests | |
| uses: ./.github/actions/run-and-record-tests | |
| with: | |
| test-subdirs: ${{ inputs.test-subdirs }} | |
| test-pattern: ${{ inputs.test-pattern }} | |
| stack-config: ${{ matrix.client-type == 'library' && 'ci-tests' || 'server:ci-tests' }} | |
| provider: ${{ matrix.provider }} | |
| inference-mode: 'replay' | |
| run-vision-tests: ${{ matrix.run-vision-tests }} |