Run Integration Tests - all libs (Python 3.10, 3.13) #73
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Routine integration tests against partner libraries with live API credentials. | |
| # | |
| # Uses `make integration_tests` within each library being tested. | |
| # | |
| # Runs daily with the option to trigger manually. | |
| name: "⏰ Integration Tests" | |
| run-name: "Run Integration Tests - ${{ inputs.working-directory-force || 'all libs' }} (Python ${{ inputs.python-version-force || '3.10, 3.13' }})" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| working-directory-force: | |
| type: string | |
| description: "From which folder this pipeline executes - defaults to all in matrix - example value: libs/partners/anthropic" | |
| python-version-force: | |
| type: string | |
| description: "Python version to use - defaults to 3.10 and 3.13 in matrix - example value: 3.11" | |
| schedule: | |
| - cron: "0 13 * * *" # Runs daily at 1PM UTC (9AM EDT/6AM PDT) | |
| permissions: | |
| contents: read | |
| env: | |
| UV_FROZEN: "true" | |
| DEFAULT_LIBS: >- | |
| ["libs/partners/openai", | |
| "libs/partners/anthropic", | |
| "libs/partners/fireworks", | |
| "libs/partners/groq", | |
| "libs/partners/mistralai", | |
| "libs/partners/xai", | |
| "libs/partners/google-vertexai", | |
| "libs/partners/google-genai", | |
| "libs/partners/aws"] | |
| jobs: | |
| # Generate dynamic test matrix based on input parameters or defaults | |
| # Only runs on the main repo (for scheduled runs) or when manually triggered | |
| compute-matrix: | |
| # Defend against forks running scheduled jobs, but allow manual runs from forks | |
| if: github.repository_owner == 'langchain-ai' || github.event_name != 'schedule' | |
| runs-on: ubuntu-latest | |
| name: "📋 Compute Test Matrix" | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| python-version-min-3-11: ${{ steps.set-matrix.outputs.python-version-min-3-11 }} | |
| steps: | |
| - name: "🔢 Generate Python & Library Matrix" | |
| id: set-matrix | |
| env: | |
| DEFAULT_LIBS: ${{ env.DEFAULT_LIBS }} | |
| WORKING_DIRECTORY_FORCE: ${{ github.event.inputs.working-directory-force || '' }} | |
| PYTHON_VERSION_FORCE: ${{ github.event.inputs.python-version-force || '' }} | |
| run: | | |
| # echo "matrix=..." where matrix is a json formatted str with keys python-version and working-directory | |
| # python-version should default to 3.10 and 3.13, but is overridden to [PYTHON_VERSION_FORCE] if set | |
| # working-directory should default to DEFAULT_LIBS, but is overridden to [WORKING_DIRECTORY_FORCE] if set | |
| python_version='["3.10", "3.13"]' | |
| python_version_min_3_11='["3.11", "3.13"]' | |
| working_directory="$DEFAULT_LIBS" | |
| if [ -n "$PYTHON_VERSION_FORCE" ]; then | |
| python_version="[\"$PYTHON_VERSION_FORCE\"]" | |
| # Bound forced version to >= 3.11 for packages requiring it | |
| if [ "$(echo "$PYTHON_VERSION_FORCE >= 3.11" | bc -l)" -eq 1 ]; then | |
| python_version_min_3_11="[\"$PYTHON_VERSION_FORCE\"]" | |
| else | |
| python_version_min_3_11='["3.11"]' | |
| fi | |
| fi | |
| if [ -n "$WORKING_DIRECTORY_FORCE" ]; then | |
| working_directory="[\"$WORKING_DIRECTORY_FORCE\"]" | |
| fi | |
| matrix="{\"python-version\": $python_version, \"working-directory\": $working_directory}" | |
| echo $matrix | |
| echo "matrix=$matrix" >> $GITHUB_OUTPUT | |
| echo "python-version-min-3-11=$python_version_min_3_11" >> $GITHUB_OUTPUT | |
| # Run integration tests against partner libraries with live API credentials | |
| integration-tests: | |
| if: github.repository_owner == 'langchain-ai' || github.event_name != 'schedule' | |
| name: "🐍 Python ${{ matrix.python-version }}: ${{ matrix.working-directory }}" | |
| runs-on: ubuntu-latest | |
| needs: [compute-matrix] | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ${{ fromJSON(needs.compute-matrix.outputs.matrix).python-version }} | |
| working-directory: ${{ fromJSON(needs.compute-matrix.outputs.matrix).working-directory }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| path: langchain | |
| # These libraries exist outside of the monorepo and need to be checked out separately | |
| - uses: actions/checkout@v6 | |
| with: | |
| repository: langchain-ai/langchain-google | |
| path: langchain-google | |
| - name: "🔐 Authenticate to Google Cloud" | |
| id: "auth" | |
| uses: google-github-actions/auth@v3 | |
| with: | |
| credentials_json: "${{ secrets.GOOGLE_CREDENTIALS }}" | |
| - uses: actions/checkout@v6 | |
| with: | |
| repository: langchain-ai/langchain-aws | |
| path: langchain-aws | |
| - name: "🔐 Configure AWS Credentials" | |
| uses: aws-actions/configure-aws-credentials@v6 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - name: "📦 Organize External Libraries" | |
| run: | | |
| rm -rf \ | |
| langchain/libs/partners/google-genai \ | |
| langchain/libs/partners/google-vertexai | |
| mv langchain-google/libs/genai langchain/libs/partners/google-genai | |
| mv langchain-google/libs/vertexai langchain/libs/partners/google-vertexai | |
| mv langchain-aws/libs/aws langchain/libs/partners/aws | |
| - name: "🐍 Set up Python ${{ matrix.python-version }} + UV" | |
| uses: "./langchain/.github/actions/uv_setup" | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: "📦 Install Dependencies" | |
| # Partner packages use [tool.uv.sources] in their pyproject.toml to resolve | |
| # langchain-core/langchain to local editable installs, so `uv sync` automatically | |
| # tests against the versions from the current branch (not published releases). | |
| # TODO: external google/aws don't have local resolution since they live in | |
| # separate repos, so they pull `core`/`langchain_v1` from PyPI. We should update | |
| # their dev groups to use git source dependencies pointing to the current | |
| # branch's latest commit SHA to fully test against local langchain changes. | |
| run: | | |
| echo "Running scheduled tests, installing dependencies with uv..." | |
| cd langchain/${{ matrix.working-directory }} | |
| uv sync --group test --group test_integration | |
| - name: "🚀 Run Integration Tests" | |
| # WARNING: All secrets below are available to every matrix job regardless of | |
| # which package is being tested. This is intentional for simplicity, but means | |
| # any test file could technically access any key. Only use for trusted code. | |
| env: | |
| LANGCHAIN_TESTS_USER_AGENT: ${{ secrets.LANGCHAIN_TESTS_USER_AGENT }} | |
| AI21_API_KEY: ${{ secrets.AI21_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| ANTHROPIC_FILES_API_IMAGE_ID: ${{ secrets.ANTHROPIC_FILES_API_IMAGE_ID }} | |
| ANTHROPIC_FILES_API_PDF_ID: ${{ secrets.ANTHROPIC_FILES_API_PDF_ID }} | |
| ASTRA_DB_API_ENDPOINT: ${{ secrets.ASTRA_DB_API_ENDPOINT }} | |
| ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN }} | |
| ASTRA_DB_KEYSPACE: ${{ secrets.ASTRA_DB_KEYSPACE }} | |
| AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }} | |
| AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} | |
| AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }} | |
| AZURE_OPENAI_LLM_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LLM_DEPLOYMENT_NAME }} | |
| AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME }} | |
| COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} | |
| DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} | |
| ES_URL: ${{ secrets.ES_URL }} | |
| ES_CLOUD_ID: ${{ secrets.ES_CLOUD_ID }} | |
| ES_API_KEY: ${{ secrets.ES_API_KEY }} | |
| EXA_API_KEY: ${{ secrets.EXA_API_KEY }} | |
| FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} | |
| GOOGLE_SEARCH_API_KEY: ${{ secrets.GOOGLE_SEARCH_API_KEY }} | |
| GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }} | |
| GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} | |
| HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} | |
| MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} | |
| MONGODB_ATLAS_URI: ${{ secrets.MONGODB_ATLAS_URI }} | |
| NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }} | |
| NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} | |
| OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| PPLX_API_KEY: ${{ secrets.PPLX_API_KEY }} | |
| TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} | |
| UPSTAGE_API_KEY: ${{ secrets.UPSTAGE_API_KEY }} | |
| WATSONX_APIKEY: ${{ secrets.WATSONX_APIKEY }} | |
| WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }} | |
| XAI_API_KEY: ${{ secrets.XAI_API_KEY }} | |
| run: | | |
| cd langchain/${{ matrix.working-directory }} | |
| make integration_tests | |
| - name: "🧹 Clean up External Libraries" | |
| # Clean up external libraries to avoid affecting the following git status check | |
| run: | | |
| rm -rf \ | |
| langchain/libs/partners/google-genai \ | |
| langchain/libs/partners/google-vertexai \ | |
| langchain/libs/partners/aws | |
| - name: "🧹 Verify Clean Working Directory" | |
| working-directory: langchain | |
| run: | | |
| set -eu | |
| STATUS="$(git status)" | |
| echo "$STATUS" | |
| # grep will exit non-zero if the target message isn't found, | |
| # and `set -e` above will cause the step to fail. | |
| echo "$STATUS" | grep 'nothing to commit, working tree clean' | |
| # Test dependent packages against local packages to catch breaking changes | |
| test-dependents: | |
| # Defend against forks running scheduled jobs, but allow manual runs from forks | |
| if: github.repository_owner == 'langchain-ai' || github.event_name != 'schedule' | |
| name: "🐍 Python ${{ matrix.python-version }}: ${{ matrix.package.path }}" | |
| runs-on: ubuntu-latest | |
| needs: [compute-matrix] | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # deepagents requires Python >= 3.11, use bounded version from compute-matrix | |
| python-version: ${{ fromJSON(needs.compute-matrix.outputs.python-version-min-3-11) }} | |
| package: | |
| - name: deepagents | |
| repo: langchain-ai/deepagents | |
| path: libs/deepagents | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| path: langchain | |
| - uses: actions/checkout@v6 | |
| with: | |
| repository: ${{ matrix.package.repo }} | |
| path: ${{ matrix.package.name }} | |
| - name: "🐍 Set up Python ${{ matrix.python-version }} + UV" | |
| uses: "./langchain/.github/actions/uv_setup" | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: "📦 Install ${{ matrix.package.name }} with Local" | |
| # Unlike partner packages (which use [tool.uv.sources] for local resolution), | |
| # external dependents live in separate repos and need explicit overrides to | |
| # test against the langchain versions from the current branch, as their | |
| # pyproject.toml files point to released versions. | |
| run: | | |
| cd ${{ matrix.package.name }}/${{ matrix.package.path }} | |
| # Install the package with test dependencies | |
| uv sync --group test | |
| # Override langchain packages with local versions | |
| uv pip install \ | |
| -e $GITHUB_WORKSPACE/langchain/libs/core \ | |
| -e $GITHUB_WORKSPACE/langchain/libs/langchain_v1 | |
| # No API keys needed for now - deepagents `make test` only runs unit tests | |
| - name: "🚀 Run ${{ matrix.package.name }} Tests" | |
| run: | | |
| cd ${{ matrix.package.name }}/${{ matrix.package.path }} | |
| make test |