Python SDK E2E Tests 7941/merge by @CometActions #6331
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python SDK E2E Tests | |
| run-name: "Python SDK E2E Tests ${{ github.ref_name }} by @${{ github.actor }}" | |
| permissions: | |
| contents: read | |
| packages: read | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| paths: | |
| - 'sdks/python/**' | |
| - 'apps/opik-backend/**' | |
| - '.github/workflows/python_sdk_e2e_tests.yml' | |
| push: | |
| branches: | |
| - 'main' | |
| paths: | |
| - 'sdks/python/**' | |
| - 'apps/opik-backend/**' | |
| - '.github/workflows/python_sdk_e2e_tests.yml' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| env: | |
| OPIK_ENABLE_LITELLM_MODELS_MONITORING: False | |
| OPIK_SENTRY_ENABLE: False | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| OPENAI_ORG_ID: ${{ secrets.OPENAI_ORG_ID }} | |
| OPIK_URL_OVERRIDE: http://localhost:8080 | |
| OPIK_CONSOLE_LOGGING_LEVEL: INFO | |
| jobs: | |
| select-matrix: | |
| name: Select Python version matrix | |
| # select-e2e-matrix lists changed files via the PR files API; without | |
| # pull-requests: read that call 403s and fails every job needing this one. | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 2 | |
| outputs: | |
| python_versions: ${{ steps.pick.outputs.versions }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| sparse-checkout: .github/actions/select-e2e-matrix | |
| sparse-checkout-cone-mode: false | |
| - uses: ./.github/actions/select-e2e-matrix | |
| id: pick | |
| with: | |
| all-versions: ${{ vars.PYTHON_VERSIONS }} | |
| match-regex: '^sdks/python/|^\.github/workflows/python_sdk_e2e_tests\.yml$' | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| build-opik: | |
| # The reusable build workflow pushes images to GHCR; pass down the | |
| # packages: write grant here rather than at the workflow level (a | |
| # called workflow's permissions can only be maintained or reduced from | |
| # the caller's, never elevated). | |
| permissions: | |
| contents: read | |
| packages: write | |
| uses: ./.github/workflows/build_e2e_docker.yaml | |
| with: | |
| include_guardrails: true | |
| run-e2e: | |
| needs: [select-matrix, build-opik] | |
| name: Python SDK E2E Tests ${{matrix.python_version}} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| # Only the "Publish Test Report" step needs these; keep them off the | |
| # workflow-level default (least privilege). | |
| permissions: | |
| contents: read | |
| packages: read | |
| checks: write | |
| pull-requests: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python_version: ${{ fromJSON(needs.select-matrix.outputs.python_versions) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Login to GHCR | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Pull Docker images from GHCR | |
| env: | |
| TAG: ${{ needs.build-opik.outputs.image_tag }} | |
| run: | | |
| docker pull "ghcr.io/comet-ml/opik/opik-backend:$TAG" | |
| docker pull "ghcr.io/comet-ml/opik/opik-python-backend:$TAG" | |
| docker pull "ghcr.io/comet-ml/opik/opik-guardrails-backend:$TAG" | |
| docker tag "ghcr.io/comet-ml/opik/opik-backend:$TAG" ghcr.io/comet-ml/opik/opik-backend:latest | |
| docker tag "ghcr.io/comet-ml/opik/opik-python-backend:$TAG" ghcr.io/comet-ml/opik/opik-python-backend:latest | |
| docker tag "ghcr.io/comet-ml/opik/opik-guardrails-backend:$TAG" ghcr.io/comet-ml/opik/opik-guardrails-backend:latest | |
| - name: Setup Python ${{matrix.python_version}} | |
| uses: actions/setup-python@v7 | |
| with: | |
| python-version: ${{matrix.python_version}} | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v9.0.0 | |
| - name: Run latest Opik server | |
| env: | |
| OPIK_USAGE_REPORT_ENABLED: false | |
| # Avoid Buildx Bake concurrent image export collisions in CI. | |
| COMPOSE_BAKE: false | |
| TOGGLE_RUNNERS_ENABLED: "true" | |
| OPIK_BACKEND_PULL_POLICY: never | |
| PYTHON_BACKEND_PULL_POLICY: never | |
| OPIK_GUARDRAILS_BACKEND_PULL_POLICY: never | |
| run: | | |
| cd ${{ github.workspace }} | |
| ./opik.sh --backend --port-mapping --guardrails | |
| - name: Check Opik server availability | |
| shell: bash | |
| run: | | |
| chmod +x ${{ github.workspace }}/tests_end_to_end/installer_utils/*.sh | |
| cd ${{ github.workspace }}/deployment/docker-compose | |
| echo "Check Docker pods are up" | |
| ${{ github.workspace }}/tests_end_to_end/installer_utils/check_docker_compose_pods.sh | |
| echo "Check backend health" | |
| ${{ github.workspace }}/tests_end_to_end/installer_utils/check_backend.sh | |
| - name: Install opik SDK | |
| run: | | |
| cd ${{ github.workspace }}/sdks/python | |
| uv pip install --system . | |
| - name: Run smoke tests | |
| run: | | |
| cd ${{ github.workspace }}/sdks/python/tests/e2e_smoke | |
| ./smoke_tests_runner.sh | |
| - name: Install test requirements | |
| run: | | |
| cd ${{ github.workspace }}/sdks/python | |
| uv pip install --system -r tests/test_requirements.txt | |
| uv pip list --system | |
| - name: Run tests | |
| # -n 3 + --dist=loadfile distributes whole test files across three | |
| # worker processes (one file per worker). Past ~3 the gains | |
| # flatten — we become tail-bound on the slowest single file and | |
| # the docker-compose backend on the same runner starts to thrash. | |
| # E2E isolation hinges on --dist=loadfile: the per-module project | |
| # name (generate_project_name("e2e", __name__)) is only collision- | |
| # free if a single file is not split across workers. | |
| # -p no:benchmark silences pytest-benchmark's auto-disable warning | |
| # under xdist. | |
| run: | | |
| cd ${{ github.workspace }}/sdks/python | |
| pytest tests/e2e --ignore=tests/e2e/test_guardrails.py --ignore=tests/e2e/compatibility_v1 -vv -n 3 --dist=loadfile -p no:benchmark --durations=20 --junitxml=${{ github.workspace }}/test_results_${{matrix.python_version}}.xml | |
| - name: Publish Test Report | |
| uses: EnricoMi/publish-unit-test-result-action/linux@v2 | |
| if: always() | |
| with: | |
| action_fail: true | |
| comment_mode: failures | |
| check_name: Python SDK E2E Tests Results (Python ${{matrix.python_version}}) | |
| files: ${{ github.workspace }}/test_results_${{matrix.python_version}}.xml | |
| - name: Keep BE log in case of failure | |
| if: failure() | |
| run: | | |
| docker logs opik-backend-1 > ${{ github.workspace }}/opik-backend_p${{matrix.python_version}}.log | |
| - name: Attach BE log | |
| if: failure() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: opik-backend-log-p${{matrix.python_version}} | |
| path: ${{ github.workspace }}/opik-backend_p${{matrix.python_version}}.log | |
| - name: Stop opik server | |
| if: always() | |
| run: | | |
| cd ${{ github.workspace }} | |
| ./opik.sh --stop --guardrails |