ci-ai.yml

name: AI
on:
    pull_request:
        types: [opened, synchronize, reopened, labeled, unlabeled]

concurrency:
    group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
    cancel-in-progress: true # We only want one AI CI run per PR concurrently

env:
    RUNS_ON_INTERNAL_PR: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false }}

jobs:
    eval:
        timeout-minutes: 45
        name: AI evals / ${{ matrix.group }}
        runs-on: ubuntu-latest
        env:
            DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USER }}
            DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
        # Skipping on forks as Braintrust credentials are not available there
        if: |
            github.repository == 'PostHog/posthog' && contains(github.event.pull_request.labels.*.name, 'evals-ready')

        strategy:
            fail-fast: false
            matrix:
                include:
                    - group: insights
                      ignore_paths: ''
                      test_paths: >-
                          ee/hogai/eval/ci/eval_funnel.py
                          ee/hogai/eval/ci/eval_trends.py
                          ee/hogai/eval/ci/eval_retention.py
                          ee/hogai/eval/ci/eval_sql.py
                          ee/hogai/eval/ci/eval_survey_analysis.py
                    - group: features
                      ignore_paths: ''
                      test_paths: >-
                          ee/hogai/eval/ci/eval_surveys.py
                          ee/hogai/eval/ci/eval_memory.py
                          ee/hogai/eval/ci/eval_memory_onboarding.py
                          ee/hogai/eval/ci/eval_ticket_summary.py
                          ee/hogai/eval/ci/eval_insight_search.py
                          ee/hogai/eval/ci/eval_ui_context.py
                    - group: root-and-tools
                      test_paths: ee/hogai/eval/ci
                      ignore_paths: >-
                          --ignore=ee/hogai/eval/ci/eval_funnel.py
                          --ignore=ee/hogai/eval/ci/eval_trends.py
                          --ignore=ee/hogai/eval/ci/eval_retention.py
                          --ignore=ee/hogai/eval/ci/eval_sql.py
                          --ignore=ee/hogai/eval/ci/eval_survey_analysis.py
                          --ignore=ee/hogai/eval/ci/eval_surveys.py
                          --ignore=ee/hogai/eval/ci/eval_memory.py
                          --ignore=ee/hogai/eval/ci/eval_memory_onboarding.py
                          --ignore=ee/hogai/eval/ci/eval_ticket_summary.py
                          --ignore=ee/hogai/eval/ci/eval_insight_search.py
                          --ignore=ee/hogai/eval/ci/eval_ui_context.py

        permissions:
            contents: read

        steps:
            - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
              with:
                  # Check out the actual branch instead of merge commit with master,
                  # because we want the Braintrust experiment to have accurate git metadata (on master it's empty)
                  ref: ${{ github.event.pull_request.head.ref }}
                  fetch-depth: 0
                  filter: blob:none
                  clean: false
            - name: Clean up data directories with container permissions
              run: |
                  # Use docker to clean up files created by containers
                  [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
              continue-on-error: true

            - name: Log in to Docker Hub
              if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}
              uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
              with:
                  username: ${{ vars.DOCKERHUB_USER }}
                  password: ${{ secrets.DOCKERHUB_TOKEN }}

            - name: Stop/Start stack with Docker Compose
              env:
                  COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml
              run: |
                  bin/ci-wait-for-docker launch --down

            - name: Wait for Docker services
              env:
                  COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml
              run: bin/ci-wait-for-docker wait

            - name: Set up Python
              uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
              with:
                  python-version-file: 'pyproject.toml'

            - name: Install uv
              uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
              with:
                  version: '0.11.28' # pinned: unpinned setup-uv calls GH API on every job, exhausts rate limit
                  enable-cache: true
                  cache-dependency-glob: uv.lock
                  save-cache: ${{ github.ref == 'refs/heads/master' }}

            - name: Install python dependencies
              shell: bash
              run: UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev

            - name: Install Rust
              uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9
              with:
                  toolchain: 1.91.1
                  components: cargo

            - name: Install sqlx-cli
              uses: ./.github/actions/setup-sqlx-cli

            - name: Add service hostnames to /etc/hosts
              run: sudo echo "127.0.0.1 db redis7 kafka clickhouse clickhouse-coordinator objectstorage seaweedfs temporal" | sudo tee -a /etc/hosts

            - name: Run LLM evals
              run: pytest ${{ matrix.test_paths }} ${{ matrix.ignore_paths }} -vv --junitxml=junit.xml
              env:
                  EVAL_MODE: ci
                  EXPORT_EVAL_RESULTS: true
                  BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
                  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
                  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
                  GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
                  INKEEP_API_KEY: ${{ secrets.INKEEP_API_KEY }}
                  AZURE_INFERENCE_CREDENTIAL: ${{ secrets.AZURE_INFERENCE_CREDENTIAL }}
                  AZURE_INFERENCE_ENDPOINT: ${{ secrets.AZURE_INFERENCE_ENDPOINT }}

            - name: Upload eval results
              uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
              if: always()
              with:
                  name: eval-results-${{ matrix.group }}
                  path: |
                      eval_results.jsonl
                      junit.xml

    eval-summary:
        timeout-minutes: 5
        name: AI evals / summary
        runs-on: ubuntu-latest
        needs: eval
        # A status function is required to summarize even when some eval groups errored.
        # !cancelled() rather than always() so a superseded run does not post an eval
        # summary comment for a commit that is no longer HEAD.
        if: ${{ !cancelled() && needs.eval.result != 'skipped' }}

        permissions:
            contents: read
            issues: write
            pull-requests: write

        steps:
            - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
              with:
                  sparse-checkout: |
                      .github/scripts
                      frontend/bin/ci-report

            - name: Download all eval results
              uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
              with:
                  pattern: eval-results-*
                  path: eval-artifacts

            - name: Merge eval results
              run: |
                  # Merge all eval_results.jsonl files into one
                  cat eval-artifacts/*/eval_results.jsonl > eval_results.jsonl 2>/dev/null || true
                  if [ ! -s eval_results.jsonl ]; then
                      echo "No eval results found"
                  else
                      echo "Merged eval results:"
                      wc -l eval_results.jsonl
                  fi

            - name: Post eval summary to CI report comment
              # always() because we want to post even if some eval groups errored
              # Skipped on trunk-merge/** branches: merge-queue draft PRs are ephemeral and
              # unread, and each section write spends shared GITHUB_TOKEN budget.
              if: always() && github.event_name == 'pull_request' && !startsWith(github.head_ref, 'trunk-merge/')
              env:
                  GITHUB_TOKEN: ${{ github.token }}
              run: |
                  if [ -f .github/scripts/post-eval-section.mjs ]; then
                      node .github/scripts/post-eval-section.mjs
                      node frontend/bin/ci-report/delete-legacy-comments.mjs
                  else
                      echo "Skipping eval section — script not present on this checkout"
                  fi

            - name: Upload merged test results
              uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
              if: always()
              with:
                  name: junit-results-ai-evals
                  path: eval-artifacts/*/junit.xml

            # Upload results to Trunk for flaky-test detection. Non-gating (continue-on-error),
            # so a Trunk outage never fails the job. Runs the token only on internal PRs
            # and master, never on fork PRs or Dependabot, which have no secret.
            # The TRUNK_UPLOAD_ENABLED repo variable is the kill-switch: when it is not 'true'
            # this upload skips.
            - name: Upload test results to Trunk
              if: ${{ !cancelled() && env.RUNS_ON_INTERNAL_PR == 'true' && github.repository == 'PostHog/posthog' && github.actor != 'dependabot[bot]' && vars.TRUNK_UPLOAD_ENABLED == 'true' }}
              continue-on-error: true
              uses: trunk-io/analytics-uploader@385f1ccdf345b4532dc4b6c665dd432b702b8e28 # v2.1.2
              with:
                  junit-paths: eval-artifacts/*/junit.xml
                  org-slug: posthog-inc
                  token: ${{ secrets.TRUNK_API_TOKEN }}