Python - Merge - Tests #163
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python - Merge - Tests | |
| # | |
| # NOTE: This workflow and python-integration-tests.yml share the same set of | |
| # parallel test jobs. Keep them in sync — when adding, removing, or modifying a | |
| # job here, apply the same change to python-integration-tests.yml. | |
| # | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| branches: ["main"] | |
| merge_group: | |
| branches: ["main"] | |
| schedule: | |
| - cron: "0 0 * * *" # Run at midnight UTC daily | |
| permissions: | |
| contents: read | |
| id-token: write | |
| env: | |
| # Configure a constant location for the uv cache | |
| UV_CACHE_DIR: /tmp/.uv-cache | |
| UV_PYTHON: "3.13" | |
| RUN_SAMPLES_TESTS: ${{ vars.RUN_SAMPLES_TESTS }} | |
| jobs: | |
| paths-filter: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| outputs: | |
| pythonChanges: ${{ steps.filter.outputs.python }} | |
| coreChanged: ${{ steps.filter.outputs.core }} | |
| openaiChanged: ${{ steps.filter.outputs.openai }} | |
| azureChanged: ${{ steps.filter.outputs.azure }} | |
| miscChanged: ${{ steps.filter.outputs.misc }} | |
| functionsChanged: ${{ steps.filter.outputs.functions }} | |
| azureAiChanged: ${{ steps.filter.outputs.azure-ai }} | |
| cosmosChanged: ${{ steps.filter.outputs.cosmos }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| filters: | | |
| python: | |
| - 'python/**' | |
| - '.github/actions/setup-local-mcp-server/**' | |
| - '.github/workflows/python-merge-tests.yml' | |
| - '.github/workflows/python-integration-tests.yml' | |
| core: | |
| - 'python/packages/core/agent_framework/_*.py' | |
| - 'python/packages/core/agent_framework/_workflows/**' | |
| - 'python/packages/core/agent_framework/exceptions.py' | |
| - 'python/packages/core/agent_framework/observability.py' | |
| openai: | |
| - 'python/packages/core/agent_framework/openai/**' | |
| - 'python/packages/openai/**' | |
| - 'python/samples/**/providers/openai/**' | |
| azure: | |
| - 'python/packages/openai/**' | |
| - 'python/packages/core/agent_framework/azure/**' | |
| - 'python/packages/azure-ai/agent_framework_azure_ai/_deprecated_azure_openai.py' | |
| - 'python/packages/azure-ai/tests/azure_openai/**' | |
| - 'python/samples/**/providers/azure/openai_chat_completion_client_azure*.py' | |
| misc: | |
| - 'python/packages/anthropic/**' | |
| - 'python/packages/ollama/**' | |
| - 'python/packages/core/agent_framework/_mcp.py' | |
| - 'python/packages/core/tests/core/test_mcp.py' | |
| - 'python/scripts/local_mcp_streamable_http_server.py' | |
| - '.github/actions/setup-local-mcp-server/**' | |
| - '.github/workflows/python-merge-tests.yml' | |
| - '.github/workflows/python-integration-tests.yml' | |
| functions: | |
| - 'python/packages/azurefunctions/**' | |
| - 'python/packages/durabletask/**' | |
| azure-ai: | |
| - 'python/packages/azure-ai/**' | |
| - 'python/packages/foundry/**' | |
| - 'python/samples/**/providers/foundry/**' | |
| cosmos: | |
| - 'python/packages/azure-cosmos/**' | |
| # run only if 'python' files were changed | |
| - name: python tests | |
| if: steps.filter.outputs.python == 'true' | |
| run: echo "Python file" | |
| # run only if not 'python' files were changed | |
| - name: not python tests | |
| if: steps.filter.outputs.python != 'true' | |
| run: echo "NOT python file" | |
| # Unit tests: always run all non-integration tests across all packages | |
| python-tests-unit: | |
| name: Python Tests - Unit | |
| needs: paths-filter | |
| if: > | |
| github.event_name != 'pull_request' && | |
| needs.paths-filter.outputs.pythonChanges == 'true' | |
| runs-on: ubuntu-latest | |
| environment: integration | |
| defaults: | |
| run: | |
| working-directory: python | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up python and install the project | |
| id: python-setup | |
| uses: ./.github/actions/python-setup | |
| with: | |
| python-version: ${{ env.UV_PYTHON }} | |
| os: ${{ runner.os }} | |
| - name: Test with pytest (unit tests only) | |
| run: > | |
| uv run poe test -A | |
| -m "not integration" | |
| --timeout=120 --session-timeout=900 --timeout_method thread | |
| --retries 2 --retry-delay 5 | |
| working-directory: ./python | |
| - name: Surface failing tests | |
| if: always() | |
| uses: pmeier/pytest-results-action@v0.7.2 | |
| with: | |
| path: ./python/**.xml | |
| summary: true | |
| display-options: fEX | |
| fail-on-empty: false | |
| title: Unit test results | |
| # OpenAI integration tests | |
| python-tests-openai: | |
| name: Python Tests - OpenAI Integration | |
| needs: paths-filter | |
| if: > | |
| github.event_name != 'pull_request' && | |
| needs.paths-filter.outputs.pythonChanges == 'true' && | |
| (github.event_name != 'merge_group' || | |
| needs.paths-filter.outputs.openaiChanged == 'true' || | |
| needs.paths-filter.outputs.coreChanged == 'true') | |
| runs-on: ubuntu-latest | |
| environment: integration | |
| env: | |
| OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }} | |
| OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }} | |
| OPENAI_EMBEDDINGS_MODEL_ID: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }} | |
| OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} | |
| OPENAI_EMBEDDING_MODEL: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} | |
| defaults: | |
| run: | |
| working-directory: python | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up python and install the project | |
| id: python-setup | |
| uses: ./.github/actions/python-setup | |
| with: | |
| python-version: ${{ env.UV_PYTHON }} | |
| os: ${{ runner.os }} | |
| - name: Test with pytest (OpenAI integration) | |
| run: > | |
| uv run pytest --import-mode=importlib | |
| packages/openai/tests | |
| -m "integration and not azure" | |
| -n logical --dist worksteal | |
| --timeout=120 --session-timeout=900 --timeout_method thread | |
| --retries 2 --retry-delay 5 | |
| working-directory: ./python | |
| - name: Test OpenAI samples | |
| timeout-minutes: 10 | |
| if: env.RUN_SAMPLES_TESTS == 'true' | |
| run: uv run pytest tests/samples/ -m "openai" | |
| working-directory: ./python | |
| - name: Surface failing tests | |
| if: always() | |
| uses: pmeier/pytest-results-action@v0.7.2 | |
| with: | |
| path: ./python/**.xml | |
| summary: true | |
| display-options: fEX | |
| fail-on-empty: false | |
| title: OpenAI integration test results | |
| # Azure OpenAI integration tests | |
| python-tests-azure-openai: | |
| name: Python Tests - Azure OpenAI Integration | |
| needs: paths-filter | |
| if: > | |
| github.event_name != 'pull_request' && | |
| needs.paths-filter.outputs.pythonChanges == 'true' && | |
| (github.event_name != 'merge_group' || | |
| needs.paths-filter.outputs.azureChanged == 'true' || | |
| needs.paths-filter.outputs.coreChanged == 'true') | |
| runs-on: ubuntu-latest | |
| environment: integration | |
| env: | |
| AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} | |
| AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} | |
| AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} | |
| AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }} | |
| AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} | |
| defaults: | |
| run: | |
| working-directory: python | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up python and install the project | |
| id: python-setup | |
| uses: ./.github/actions/python-setup | |
| with: | |
| python-version: ${{ env.UV_PYTHON }} | |
| os: ${{ runner.os }} | |
| - name: Azure CLI Login | |
| if: github.event_name != 'pull_request' | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| - name: Test with pytest (Azure OpenAI integration) | |
| run: > | |
| uv run pytest --import-mode=importlib | |
| packages/openai/tests/openai/test_openai_chat_completion_client_azure.py | |
| packages/openai/tests/openai/test_openai_chat_client_azure.py | |
| packages/azure-ai/tests/azure_openai | |
| -m integration | |
| -n logical --dist worksteal | |
| --timeout=120 --session-timeout=900 --timeout_method thread | |
| --retries 2 --retry-delay 5 | |
| working-directory: ./python | |
| - name: Test Azure samples | |
| timeout-minutes: 10 | |
| if: env.RUN_SAMPLES_TESTS == 'true' | |
| run: uv run pytest tests/samples/ -m "azure" | |
| working-directory: ./python | |
| - name: Surface failing tests | |
| if: always() | |
| uses: pmeier/pytest-results-action@v0.7.2 | |
| with: | |
| path: ./python/**.xml | |
| summary: true | |
| display-options: fEX | |
| fail-on-empty: false | |
| title: Azure OpenAI integration test results | |
| # Misc integration tests (Anthropic, Ollama, MCP) | |
| python-tests-misc-integration: | |
| name: Python Tests - Misc Integration | |
| needs: paths-filter | |
| if: > | |
| github.event_name != 'pull_request' && | |
| needs.paths-filter.outputs.pythonChanges == 'true' && | |
| (github.event_name != 'merge_group' || | |
| needs.paths-filter.outputs.miscChanged == 'true' || | |
| needs.paths-filter.outputs.coreChanged == 'true') | |
| runs-on: ubuntu-latest | |
| environment: integration | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| ANTHROPIC_CHAT_MODEL_ID: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }} | |
| LOCAL_MCP_URL: ${{ vars.LOCAL_MCP__URL }} | |
| defaults: | |
| run: | |
| working-directory: python | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up python and install the project | |
| id: python-setup | |
| uses: ./.github/actions/python-setup | |
| with: | |
| python-version: ${{ env.UV_PYTHON }} | |
| os: ${{ runner.os }} | |
| - name: Start local MCP server | |
| id: local-mcp | |
| uses: ./.github/actions/setup-local-mcp-server | |
| with: | |
| fallback_url: ${{ env.LOCAL_MCP_URL }} | |
| - name: Prefer local MCP URL when available | |
| run: echo "LOCAL_MCP_URL=${{ steps.local-mcp.outputs.effective_url }}" >> "$GITHUB_ENV" | |
| - name: Test with pytest (Anthropic, Ollama, MCP integration) | |
| run: > | |
| uv run pytest --import-mode=importlib | |
| packages/anthropic/tests | |
| packages/ollama/tests | |
| packages/core/tests/core/test_mcp.py | |
| -m integration | |
| -n logical --dist worksteal | |
| --timeout=120 --session-timeout=900 --timeout_method thread | |
| --retries 2 --retry-delay 5 | |
| working-directory: ./python | |
| - name: Stop local MCP server | |
| if: always() | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| server_pid="${{ steps.local-mcp.outputs.pid }}" | |
| if [[ -z "$server_pid" ]]; then | |
| exit 0 | |
| fi | |
| if ! kill -0 "$server_pid" 2>/dev/null; then | |
| exit 0 | |
| fi | |
| kill -TERM -- "-$server_pid" 2>/dev/null || kill -TERM "$server_pid" 2>/dev/null || true | |
| for _ in $(seq 1 10); do | |
| if ! kill -0 "$server_pid" 2>/dev/null; then | |
| exit 0 | |
| fi | |
| sleep 1 | |
| done | |
| kill -KILL -- "-$server_pid" 2>/dev/null || kill -KILL "$server_pid" 2>/dev/null || true | |
| - name: Surface failing tests | |
| if: always() | |
| uses: pmeier/pytest-results-action@v0.7.2 | |
| with: | |
| path: ./python/**.xml | |
| summary: true | |
| display-options: fEX | |
| fail-on-empty: false | |
| title: Misc integration test results | |
| # Azure Functions + Durable Task integration tests | |
| python-tests-functions: | |
| name: Python Tests - Functions Integration | |
| needs: paths-filter | |
| if: > | |
| github.event_name != 'pull_request' && | |
| needs.paths-filter.outputs.pythonChanges == 'true' && | |
| (github.event_name != 'merge_group' || | |
| needs.paths-filter.outputs.functionsChanged == 'true' || | |
| needs.paths-filter.outputs.coreChanged == 'true') | |
| runs-on: ubuntu-latest | |
| environment: integration | |
| env: | |
| UV_PYTHON: "3.11" | |
| OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }} | |
| OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }} | |
| OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} | |
| OPENAI_EMBEDDING_MODEL: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }} | |
| AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} | |
| AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} | |
| FOUNDRY_MODEL: ${{ vars.AZUREAI__DEPLOYMENTNAME }} | |
| FOUNDRY_PROJECT_ENDPOINT: ${{ secrets.AZUREAI__ENDPOINT }} | |
| FUNCTIONS_WORKER_RUNTIME: "python" | |
| DURABLE_TASK_SCHEDULER_CONNECTION_STRING: "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" | |
| AzureWebJobsStorage: "UseDevelopmentStorage=true" | |
| defaults: | |
| run: | |
| working-directory: python | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up python and install the project | |
| id: python-setup | |
| uses: ./.github/actions/python-setup | |
| with: | |
| python-version: ${{ env.UV_PYTHON }} | |
| os: ${{ runner.os }} | |
| - name: Azure CLI Login | |
| if: github.event_name != 'pull_request' | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| - name: Set up Azure Functions Integration Test Emulators | |
| uses: ./.github/actions/azure-functions-integration-setup | |
| id: azure-functions-setup | |
| - name: Test with pytest (Functions + Durable Task integration) | |
| run: > | |
| uv run pytest --import-mode=importlib | |
| packages/azurefunctions/tests/integration_tests | |
| packages/durabletask/tests/integration_tests | |
| -m integration | |
| -n logical --dist worksteal | |
| -x | |
| --timeout=360 --session-timeout=900 --timeout_method thread | |
| --retries 2 --retry-delay 5 | |
| working-directory: ./python | |
| - name: Surface failing tests | |
| if: always() | |
| uses: pmeier/pytest-results-action@v0.7.2 | |
| with: | |
| path: ./python/**.xml | |
| summary: true | |
| display-options: fEX | |
| fail-on-empty: false | |
| title: Functions integration test results | |
| python-tests-azure-ai: | |
| name: Python Tests - Azure AI | |
| needs: paths-filter | |
| if: > | |
| github.event_name != 'pull_request' && | |
| needs.paths-filter.outputs.pythonChanges == 'true' && | |
| (github.event_name != 'merge_group' || | |
| needs.paths-filter.outputs.azureAiChanged == 'true' || | |
| needs.paths-filter.outputs.coreChanged == 'true') | |
| runs-on: ubuntu-latest | |
| environment: integration | |
| env: | |
| AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZUREAI__ENDPOINT }} | |
| AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREAI__DEPLOYMENTNAME }} | |
| FOUNDRY_PROJECT_ENDPOINT: ${{ secrets.AZUREAI__ENDPOINT }} | |
| FOUNDRY_MODEL: ${{ vars.AZUREAI__DEPLOYMENTNAME }} | |
| LOCAL_MCP_URL: ${{ vars.LOCAL_MCP__URL }} | |
| defaults: | |
| run: | |
| working-directory: python | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up python and install the project | |
| id: python-setup | |
| uses: ./.github/actions/python-setup | |
| with: | |
| python-version: ${{ env.UV_PYTHON }} | |
| os: ${{ runner.os }} | |
| - name: Azure CLI Login | |
| if: github.event_name != 'pull_request' | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| - name: Test with pytest | |
| timeout-minutes: 15 | |
| run: | | |
| uv run --directory packages/azure-ai poe integration-tests -n logical --dist worksteal --timeout=120 --session-timeout=900 --timeout_method thread --retries 2 --retry-delay 5 | |
| uv run --directory packages/foundry poe integration-tests -n logical --dist worksteal --timeout=120 --session-timeout=900 --timeout_method thread --retries 2 --retry-delay 5 | |
| working-directory: ./python | |
| - name: Test Azure AI samples | |
| timeout-minutes: 10 | |
| if: env.RUN_SAMPLES_TESTS == 'true' | |
| run: uv run pytest tests/samples/ -m "azure-ai" | |
| working-directory: ./python | |
| - name: Surface failing tests | |
| if: always() | |
| uses: pmeier/pytest-results-action@v0.7.2 | |
| with: | |
| path: ./python/**.xml | |
| summary: true | |
| display-options: fEX | |
| fail-on-empty: false | |
| title: Test results | |
| # TODO: Add python-tests-lab | |
| # Azure Cosmos integration tests | |
| python-tests-cosmos: | |
| name: Python Tests - Cosmos Integration | |
| needs: paths-filter | |
| if: > | |
| github.event_name != 'pull_request' && | |
| needs.paths-filter.outputs.pythonChanges == 'true' && | |
| (github.event_name != 'merge_group' || | |
| needs.paths-filter.outputs.cosmosChanged == 'true' || | |
| needs.paths-filter.outputs.coreChanged == 'true') | |
| runs-on: ubuntu-latest | |
| environment: integration | |
| services: | |
| cosmosdb: | |
| image: mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-preview | |
| ports: | |
| - 8081:8081 | |
| env: | |
| AZURE_COSMOS_ENDPOINT: "http://localhost:8081/" | |
| # Static Azure Cosmos DB emulator key (documented): https://learn.microsoft.com/en-us/azure/cosmos-db/emulator | |
| AZURE_COSMOS_KEY: "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==" | |
| AZURE_COSMOS_DATABASE_NAME: "agent-framework-cosmos-it-db" | |
| AZURE_COSMOS_CONTAINER_NAME: "agent-framework-cosmos-it-container" | |
| defaults: | |
| run: | |
| working-directory: python | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up python and install the project | |
| id: python-setup | |
| uses: ./.github/actions/python-setup | |
| with: | |
| python-version: ${{ env.UV_PYTHON }} | |
| os: ${{ runner.os }} | |
| - name: Wait for Cosmos DB emulator | |
| run: | | |
| for i in {1..60}; do | |
| if curl --silent --show-error http://localhost:8081/ > /dev/null; then | |
| echo "Cosmos DB emulator is ready." | |
| exit 0 | |
| fi | |
| sleep 2 | |
| done | |
| echo "Cosmos DB emulator did not become ready in time." >&2 | |
| exit 1 | |
| - name: Test with pytest (Cosmos integration) | |
| run: uv run --directory packages/azure-cosmos poe integration-tests -n logical --dist worksteal --timeout=120 --session-timeout=900 --timeout_method thread --retries 2 --retry-delay 5 | |
| working-directory: ./python | |
| - name: Surface failing tests | |
| if: always() | |
| uses: pmeier/pytest-results-action@v0.7.2 | |
| with: | |
| path: ./python/**.xml | |
| summary: true | |
| display-options: fEX | |
| fail-on-empty: false | |
| title: Cosmos integration test results | |
| python-integration-tests-check: | |
| if: always() | |
| runs-on: ubuntu-latest | |
| needs: | |
| [ | |
| python-tests-unit, | |
| python-tests-openai, | |
| python-tests-azure-openai, | |
| python-tests-misc-integration, | |
| python-tests-functions, | |
| python-tests-azure-ai, | |
| python-tests-cosmos, | |
| ] | |
| steps: | |
| - name: Fail workflow if tests failed | |
| id: check_tests_failed | |
| if: contains(join(needs.*.result, ','), 'failure') | |
| uses: actions/github-script@v8 | |
| with: | |
| script: core.setFailed('Integration Tests Failed!') | |
| - name: Fail workflow if tests cancelled | |
| id: check_tests_cancelled | |
| if: contains(join(needs.*.result, ','), 'cancelled') | |
| uses: actions/github-script@v8 | |
| with: | |
| script: core.setFailed('Integration Tests Cancelled!') |