Skip to content
Test

fix(security): add token budget limiting and LLM validator sanitization #5563

Sign in to view logs

fix(security): add token budget limiting and LLM validator sanitization

fix(security): add token budget limiting and LLM validator sanitization #5563

Workflow file for this run

.github/workflows/test.yml at 5dad44a
name: Test
on:
pull_request:
push:
branches:
- main
jobs:
# Core tests without LLM providers
core-tests:
name: Core Tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Install uv
uses: astral-sh/setup-uv@v4
with:
enable-cache: true
- name: Set up Python
run: uv python install 3.11
- name: Install the project
run: uv sync --all-extras
- name: Run core tests
run: >-
uv run pytest tests/ --asyncio-mode=auto -n auto
-k 'not test_core_providers and not test_openai and not test_anthropic
and not test_gemini and not test_genai and not test_writer and not
test_vertexai and not docs'
env:
INSTRUCTOR_ENV: CI
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
# Core provider tests for OpenAI
core-openai:
name: Core Provider Tests (OpenAI)
runs-on: ubuntu-latest
needs: core-tests
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
steps:
- uses: actions/checkout@v2
- name: Install uv
uses: astral-sh/setup-uv@v4
with:
enable-cache: true
- name: Set up Python
run: uv python install 3.11
- name: Install the project
run: uv sync --all-extras
- name: Skip core provider tests (OpenAI)
if: ${{ env.OPENAI_API_KEY == '' }}
run: echo "Skipping OpenAI core provider tests (missing OPENAI_API_KEY)."
- name: Run core provider tests (OpenAI)
if: ${{ env.OPENAI_API_KEY != '' }}
run: |
set +e
uv run pytest tests/llm/test_core_providers -v --asyncio-mode=auto -n auto -k "openai"
status=$?
set -e
if [ $status -eq 5 ]; then
echo "No tests collected; treating as success."
exit 0
fi
exit $status
env:
INSTRUCTOR_ENV: CI
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
# Core provider tests for Anthropic
core-anthropic:
name: Core Provider Tests (Anthropic)
runs-on: ubuntu-latest
needs: core-tests
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
steps:
- uses: actions/checkout@v2
- name: Install uv
uses: astral-sh/setup-uv@v4
with:
enable-cache: true
- name: Set up Python
run: uv python install 3.11
- name: Install the project
run: uv sync --all-extras
- name: Skip core provider tests (Anthropic)
if: ${{ env.ANTHROPIC_API_KEY == '' }}
run: echo "Skipping Anthropic core provider tests (missing ANTHROPIC_API_KEY)."
- name: Run core provider tests (Anthropic)
if: ${{ env.ANTHROPIC_API_KEY != '' }}
run: |
set +e
uv run pytest tests/llm/test_core_providers -v --asyncio-mode=auto -n auto -k "anthropic"
status=$?
set -e
if [ $status -eq 5 ]; then
echo "No tests collected; treating as success."
exit 0
fi
exit $status
env:
INSTRUCTOR_ENV: CI
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
# Core provider tests for Google
core-google:
name: Core Provider Tests (Google)
runs-on: ubuntu-latest
needs: core-tests
env:
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
GOOGLE_GENAI_MODEL: ${{ secrets.GOOGLE_GENAI_MODEL }}
steps:
- uses: actions/checkout@v2
- name: Install uv
uses: astral-sh/setup-uv@v4
with:
enable-cache: true
- name: Set up Python
run: uv python install 3.11
- name: Install the project
run: uv sync --all-extras
- name: Skip core provider tests (Google)
if: ${{ env.GOOGLE_API_KEY == '' || env.GOOGLE_GENAI_MODEL == '' }}
run: echo "Skipping Google core provider tests (missing GOOGLE_API_KEY or GOOGLE_GENAI_MODEL)."
- name: Run core provider tests (Google)
if: ${{ env.GOOGLE_API_KEY != '' && env.GOOGLE_GENAI_MODEL != '' }}
run: |
set +e
uv run pytest tests/llm/test_core_providers -v --asyncio-mode=auto -n auto -k "google"
status=$?
set -e
if [ $status -eq 5 ]; then
echo "No tests collected; treating as success."
exit 0
fi
exit $status
env:
INSTRUCTOR_ENV: CI
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
# Core provider tests for other providers
core-other:
name: Core Provider Tests (Other)
runs-on: ubuntu-latest
needs: core-tests
env:
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }}
FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
WRITER_API_KEY: ${{ secrets.WRITER_API_KEY }}
PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }}
steps:
- uses: actions/checkout@v2
- name: Install uv
uses: astral-sh/setup-uv@v4
with:
enable-cache: true
- name: Set up Python
run: uv python install 3.11
- name: Install the project
run: uv sync --all-extras
- name: Skip core provider tests (Other)
if: >-
${{ env.COHERE_API_KEY == '' && env.XAI_API_KEY == ''
&& env.MISTRAL_API_KEY == '' && env.CEREBRAS_API_KEY == ''
&& env.FIREWORKS_API_KEY == '' && env.WRITER_API_KEY == ''
&& env.PERPLEXITY_API_KEY == '' }}
run: echo "Skipping core provider tests (Other) (missing provider secrets)."
- name: Run core provider tests (Cohere, xAI, Mistral, etc)
if: >-
${{ env.COHERE_API_KEY != '' || env.XAI_API_KEY != ''
|| env.MISTRAL_API_KEY != '' || env.CEREBRAS_API_KEY != ''
|| env.FIREWORKS_API_KEY != '' || env.WRITER_API_KEY != ''
|| env.PERPLEXITY_API_KEY != '' }}
run: |
set +e
uv run pytest tests/llm/test_core_providers -v --asyncio-mode=auto -n auto -k "cohere or xai or mistral or cerebras or fireworks or writer or perplexity"
status=$?
set -e
if [ $status -eq 5 ]; then
echo "No tests collected; treating as success."
exit 0
fi
exit $status
env:
INSTRUCTOR_ENV: CI
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }}
FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
WRITER_API_KEY: ${{ secrets.WRITER_API_KEY }}
PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }}
# Provider tests run in parallel
provider-tests:
name: ${{ matrix.provider.name }} Tests
runs-on: ubuntu-latest
needs: [core-openai, core-anthropic, core-google, core-other]
env:
PROVIDER_API_KEY: ${{ secrets[matrix.provider.env_key] }}
GOOGLE_GENAI_MODEL: ${{ secrets.GOOGLE_GENAI_MODEL }}
strategy:
fail-fast: false
matrix:
provider:
- name: OpenAI
env_key: OPENAI_API_KEY
test_path: tests/llm/test_openai
- name: Anthropic
env_key: ANTHROPIC_API_KEY
test_path: tests/llm/test_anthropic
- name: Gemini
env_key: GOOGLE_API_KEY
test_path: tests/llm/test_gemini
- name: Google GenAI
env_key: GOOGLE_API_KEY
test_path: tests/llm/test_genai
- name: Vertex AI
env_key: GOOGLE_API_KEY
test_path: tests/llm/test_vertexai
- name: Writer
env_key: WRITER_API_KEY
test_path: tests/llm/test_writer
steps:
- uses: actions/checkout@v2
- name: Install uv
uses: astral-sh/setup-uv@v4
with:
enable-cache: true
- name: Set up Python
run: uv python install 3.11
- name: Install the project
run: uv sync --all-extras
- name: Skip ${{ matrix.provider.name }} tests
if: >-
${{ env.PROVIDER_API_KEY == '' ||
((matrix.provider.name == 'Gemini' || matrix.provider.name == 'Google GenAI'
|| matrix.provider.name == 'Vertex AI') && env.GOOGLE_GENAI_MODEL == '') }}
run: >-
echo "Skipping ${{ matrix.provider.name }} tests
(missing ${{ matrix.provider.env_key }} or GOOGLE_GENAI_MODEL)."
- name: Run ${{ matrix.provider.name }} tests
if: >-
${{ env.PROVIDER_API_KEY != '' &&
((matrix.provider.name != 'Gemini' && matrix.provider.name != 'Google GenAI'
&& matrix.provider.name != 'Vertex AI') || env.GOOGLE_GENAI_MODEL != '') }}
run: |
set +e
uv run pytest ${{ matrix.provider.test_path }} --asyncio-mode=auto -n auto
status=$?
set -e
if [ $status -eq 5 ]; then
echo "No tests collected; treating as success."
exit 0
fi
exit $status
env:
INSTRUCTOR_ENV: CI
${{ matrix.provider.env_key }}: ${{ secrets[matrix.provider.env_key] }}
# Auto client needs multiple providers
auto-client-test:
name: Auto Client Tests
runs-on: ubuntu-latest
needs: [core-openai, core-anthropic, core-google, core-other]
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
steps:
- uses: actions/checkout@v2
- name: Install uv
uses: astral-sh/setup-uv@v4
with:
enable-cache: true
- name: Set up Python
run: uv python install 3.11
- name: Install the project
run: uv sync --all-extras
- name: Skip Auto Client tests
if: >-
${{ env.OPENAI_API_KEY == '' || env.GOOGLE_API_KEY == ''
|| env.COHERE_API_KEY == '' || env.ANTHROPIC_API_KEY == ''
|| env.XAI_API_KEY == '' }}
run: echo "Skipping Auto Client tests (missing one or more provider secrets)."
- name: Run Auto Client tests
if: >-
${{ env.OPENAI_API_KEY != '' && env.GOOGLE_API_KEY != ''
&& env.COHERE_API_KEY != '' && env.ANTHROPIC_API_KEY != ''
&& env.XAI_API_KEY != '' }}
run: |
set +e
uv run pytest tests/test_auto_client.py --asyncio-mode=auto -n auto
status=$?
set -e
if [ $status -eq 5 ]; then
echo "No tests collected; treating as success."
exit 0
fi
exit $status
env:
INSTRUCTOR_ENV: CI
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}