pr-python-connector-tests.yml

name: Connector Tests

# Connector tests run INSIDE the production backend image (backend/Dockerfile)
# rather than on a bare runner. A bare runner installs Playwright with
# `--with-deps`, so it never exercises the system libraries actually shipped in
# the image -- which let an image-only regression slip through (a missing
# `libnss3.so`, chrome-headless-shell failing to load shared libraries after the
# Dockerfile's apt autoremove cascade). Running in-image catches those.
#
# These tests run on presubmit but are non-blocking, so the extra image build +
# in-container setup is an acceptable cost for the added correctness. The build
# is registry-cached: PRs that don't touch the Dockerfile or requirements get a
# near-instant cached build.

concurrency:
  group: Connector-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
  cancel-in-progress: true

on:
  # Connector tests are non-blocking (not a required status check) and exercise
  # real external connector APIs, so they don't belong in the merge queue.
  # Running on `merge_group` only exposed them to the ephemeral
  # `gh-readonly-queue/*` branch-deletion race in paths-filter -- when an entry
  # ahead in the queue fails and GitHub tears down/recreates the queue branches
  # mid-run, paths-filter's fetch of the head ref by name fails and the whole job
  # goes red for a reason unrelated to the change under test -- without gating any
  # merge. Run on PRs for pre-merge signal and on push-to-main for post-merge
  # detection of whatever actually landed; `main` is a permanent ref so the race
  # can't happen. The daily schedule stays the full-coverage safety net. `paths`
  # is not evaluated for tag pushes, so the `v*.*.*` release trigger below still
  # fires regardless of changed files.
  pull_request:
    branches: [main]
    paths:
      - "backend/**"
      - "pyproject.toml"
      - "uv.lock"
      - ".github/workflows/pr-python-connector-tests.yml"
      - ".github/actions/build-backend-image/**"
      - ".github/actions/login-ecr-pullthrough-cache/**"
  push:
    branches:
      - main
    paths:
      - "backend/**"
      - "pyproject.toml"
      - "uv.lock"
      - ".github/workflows/pr-python-connector-tests.yml"
      - ".github/actions/build-backend-image/**"
      - ".github/actions/login-ecr-pullthrough-cache/**"
    tags:
      - "v*.*.*"
  schedule:
    # This cron expression runs the job daily at 16:00 UTC (9am PT)
    - cron: "0 16 * * *"

permissions:
  contents: read

env:
  PYTHONPATH: ./backend
  DISABLE_TELEMETRY: "true"
  R2_ACCOUNT_ID_DAILY_CONNECTOR_TESTS: ${{ vars.R2_ACCOUNT_ID_DAILY_CONNECTOR_TESTS }}
  CONFLUENCE_TEST_SPACE_URL: ${{ vars.CONFLUENCE_TEST_SPACE_URL }}
  CONFLUENCE_TEST_SPACE: ${{ vars.CONFLUENCE_TEST_SPACE }}
  CONFLUENCE_USER_NAME: ${{ vars.CONFLUENCE_USER_NAME }}
  SF_USERNAME: ${{ vars.SF_USERNAME }}
  IMAP_HOST: ${{ vars.IMAP_HOST }}
  IMAP_USERNAME: ${{ vars.IMAP_USERNAME }}
  IMAP_MAILBOXES: ${{ vars.IMAP_MAILBOXES }}
  AIRTABLE_TEST_BASE_ID: ${{ vars.AIRTABLE_TEST_BASE_ID }}
  AIRTABLE_TEST_TABLE_ID: ${{ vars.AIRTABLE_TEST_TABLE_ID }}
  AIRTABLE_TEST_TABLE_NAME: ${{ vars.AIRTABLE_TEST_TABLE_NAME }}
  SHAREPOINT_CLIENT_ID: ${{ vars.SHAREPOINT_CLIENT_ID }}
  SHAREPOINT_CLIENT_DIRECTORY_ID: ${{ vars.SHAREPOINT_CLIENT_DIRECTORY_ID }}
  SHAREPOINT_SITE: ${{ vars.SHAREPOINT_SITE }}
  BITBUCKET_EMAIL: ${{ vars.BITBUCKET_EMAIL }}

jobs:
  build-backend-image:
    # See https://runs-on.com/runners/linux/
    runs-on:
      [
        runs-on,
        runner=8cpu-linux-x64,
        "run-id=${{ github.run_id }}-build-backend-image",
        "extras=ecr-cache",
      ]
    timeout-minutes: 10
    environment: ci-protected

    steps:
      - uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc

      - name: Checkout code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
        with:
          persist-credentials: false

      - name: Build backend image
        uses: ./.github/actions/build-backend-image
        with:
          runs-on-ecr-cache: ${{ env.RUNS_ON_ECR_CACHE }}
          ref-name: ${{ github.ref_name }}
          pr-number: ${{ github.event.pull_request.number }}
          github-sha: ${{ github.sha }}
          run-id: ${{ github.run_id }}
          ecr-registry: ${{ vars.ECR_REGISTRY }}
          docker-no-cache: ${{ vars.DOCKER_NO_CACHE == 'true' && 'true' || 'false' }}

  connectors-check:
    needs: build-backend-image
    # id-token scoped to this job only -- it's the sole job that exchanges OIDC for AWS
    # credentials (to fetch connector secrets); build-backend-image authenticates to ECR via
    # the runner instance role instead.
    permissions:
      id-token: write
      contents: read
      # paths-filter needs pull-requests:read to list PR files on private repos (no-op on public).
      pull-requests: read
    # See https://runs-on.com/runners/linux/
    runs-on:
      [
        runs-on,
        runner=8cpu-linux-x64,
        "run-id=${{ github.run_id }}-connectors-check",
        "extras=ecr-cache",
      ]
    timeout-minutes: 40
    environment: ci-protected

    steps:
      - uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc

      - name: Checkout code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
        with:
          persist-credentials: false

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # ratchet:aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
          aws-region: us-east-2

      - name: Detect Connector changes
        id: changes
        uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706
        with:
          filters: |
            hubspot:
              - 'backend/onyx/connectors/hubspot/**'
              - 'backend/tests/daily/connectors/hubspot/**'
              - 'uv.lock'
            salesforce:
              - 'backend/onyx/connectors/salesforce/**'
              - 'backend/tests/daily/connectors/salesforce/**'
              - 'uv.lock'
            github:
              - 'backend/onyx/connectors/github/**'
              - 'backend/tests/daily/connectors/github/**'
              - 'uv.lock'
            file_processing:
              - 'backend/onyx/file_processing/**'
              - 'uv.lock'

      - name: Pull production backend image
        env:
          PROD_IMAGE: ${{ env.RUNS_ON_ECR_CACHE }}:nightly-llm-it-backend-${{ github.run_id }}
        run: docker pull "${PROD_IMAGE}"

      # Start a long-lived container from the production image and layer in the
      # test-only deps (pytest, etc.) that aren't baked into it -- the prod image
      # installs only default.txt + ee.txt and does not COPY ./tests. We install
      # against the system interpreter (where the image's deps already live) and
      # bind-mount the checked-out workspace for the test files. The Playwright
      # browser binary and the system libraries it links against (libnss3, etc.)
      # come from the image -- that is the thing under test. AWS credentials are
      # baked into the container env so the pytest_secrets fixture can batch-fetch
      # connector credentials from Secrets Manager.
      - name: Start test container
        env:
          PROD_IMAGE: ${{ env.RUNS_ON_ECR_CACHE }}:nightly-llm-it-backend-${{ github.run_id }}
        run: |
          docker run -d --name onyx-connector-tests --user root \
            -v "${{ github.workspace }}:/workspace" \
            -w /workspace \
            -e PYTHONPATH=./backend \
            -e DISABLE_TELEMETRY \
            -e AWS_ACCESS_KEY_ID \
            -e AWS_SECRET_ACCESS_KEY \
            -e AWS_SESSION_TOKEN \
            -e AWS_REGION \
            -e AWS_DEFAULT_REGION \
            -e R2_ACCOUNT_ID_DAILY_CONNECTOR_TESTS \
            -e CONFLUENCE_TEST_SPACE_URL \
            -e CONFLUENCE_TEST_SPACE \
            -e CONFLUENCE_USER_NAME \
            -e SF_USERNAME \
            -e IMAP_HOST \
            -e IMAP_USERNAME \
            -e IMAP_MAILBOXES \
            -e AIRTABLE_TEST_BASE_ID \
            -e AIRTABLE_TEST_TABLE_ID \
            -e AIRTABLE_TEST_TABLE_NAME \
            -e SHAREPOINT_CLIENT_ID \
            -e SHAREPOINT_CLIENT_DIRECTORY_ID \
            -e SHAREPOINT_SITE \
            -e BITBUCKET_EMAIL \
            "${PROD_IMAGE}" sleep infinity
          # The prod image no longer ships uv, so install the test-only deps
          # (pytest, etc.) with the image's own pip against the system
          # interpreter, where the image's deps already live. pip ignores
          # pyproject.toml's [tool.uv] overrides, so --require-hashes resolves
          # directly without uv's --no-config workaround.
          docker exec onyx-connector-tests bash -c '
            set -euo pipefail
            python -m pip install --no-deps --require-hashes \
              -r backend/requirements/default.txt \
              -r backend/requirements/dev.txt
          '

      # The prod image has no uv or .venv; run pytest with the system
      # interpreter directly, where the deps installed above live.
      - name: Run Tests (excluding HubSpot, Salesforce, GitHub, and Coda)
        shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
        run: |
          docker exec onyx-connector-tests bash -c '
            python -m pytest \
              -n 8 \
              --dist loadfile \
              --durations=8 \
              -o junit_family=xunit2 \
              -xv \
              --ff \
              backend/tests/daily/connectors \
              --ignore backend/tests/daily/connectors/hubspot \
              --ignore backend/tests/daily/connectors/salesforce \
              --ignore backend/tests/daily/connectors/github \
              --ignore backend/tests/daily/connectors/coda
          '

      - name: Run HubSpot Connector Tests
        if: ${{ github.event_name == 'schedule' || steps.changes.outputs.hubspot == 'true' || steps.changes.outputs.file_processing == 'true' }}
        shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
        run: |
          docker exec onyx-connector-tests bash -c '
            python -m pytest \
              -n 8 \
              --dist loadfile \
              --durations=8 \
              -o junit_family=xunit2 \
              -xv \
              --ff \
              backend/tests/daily/connectors/hubspot
          '

      - name: Run Salesforce Connector Tests
        if: ${{ github.event_name == 'schedule' || steps.changes.outputs.salesforce == 'true' || steps.changes.outputs.file_processing == 'true' }}
        shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
        run: |
          docker exec onyx-connector-tests bash -c '
            python -m pytest \
              -n 8 \
              --dist loadfile \
              --durations=8 \
              -o junit_family=xunit2 \
              -xv \
              --ff \
              backend/tests/daily/connectors/salesforce
          '

      - name: Run GitHub Connector Tests
        if: ${{ github.event_name == 'schedule' || steps.changes.outputs.github == 'true' || steps.changes.outputs.file_processing == 'true' }}
        shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
        run: |
          docker exec onyx-connector-tests bash -c '
            python -m pytest \
              -n 8 \
              --dist loadfile \
              --durations=8 \
              -o junit_family=xunit2 \
              -xv \
              --ff \
              backend/tests/daily/connectors/github
          '

      - name: Stop test container
        if: always()
        run: docker rm -f onyx-connector-tests || true

      - name: Alert on Failure
        if: failure() && github.event_name == 'schedule'
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          REPO: ${{ github.repository }}
          RUN_ID: ${{ github.run_id }}
        run: |
          curl -X POST \
            -H 'Content-type: application/json' \
            --data "{\"text\":\"Scheduled Connector Tests failed! Check the run at: https://github.com/${REPO}/actions/runs/${RUN_ID}\"}" \
            $SLACK_WEBHOOK