fix(signals): evict stale db connections in temporal activities #133939
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: AI | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, labeled, unlabeled] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: true # We only want one AI CI run per PR concurrently | |
| jobs: | |
| eval: | |
| timeout-minutes: 45 | |
| name: AI evals / ${{ matrix.group }} | |
| runs-on: ubuntu-latest | |
| # Skipping on forks as Braintrust credentials are not available there | |
| if: | | |
| github.repository == 'PostHog/posthog' && contains(github.event.pull_request.labels.*.name, 'evals-ready') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - group: insights | |
| ignore_paths: '' | |
| test_paths: >- | |
| ee/hogai/eval/ci/eval_funnel.py | |
| ee/hogai/eval/ci/eval_trends.py | |
| ee/hogai/eval/ci/eval_retention.py | |
| ee/hogai/eval/ci/eval_sql.py | |
| ee/hogai/eval/ci/eval_survey_analysis.py | |
| - group: features | |
| ignore_paths: '' | |
| test_paths: >- | |
| ee/hogai/eval/ci/eval_surveys.py | |
| ee/hogai/eval/ci/eval_memory.py | |
| ee/hogai/eval/ci/eval_memory_onboarding.py | |
| ee/hogai/eval/ci/eval_ticket_summary.py | |
| ee/hogai/eval/ci/eval_insight_search.py | |
| ee/hogai/eval/ci/eval_ui_context.py | |
| - group: root-and-tools | |
| test_paths: ee/hogai/eval/ci | |
| ignore_paths: >- | |
| --ignore=ee/hogai/eval/ci/eval_funnel.py | |
| --ignore=ee/hogai/eval/ci/eval_trends.py | |
| --ignore=ee/hogai/eval/ci/eval_retention.py | |
| --ignore=ee/hogai/eval/ci/eval_sql.py | |
| --ignore=ee/hogai/eval/ci/eval_survey_analysis.py | |
| --ignore=ee/hogai/eval/ci/eval_surveys.py | |
| --ignore=ee/hogai/eval/ci/eval_memory.py | |
| --ignore=ee/hogai/eval/ci/eval_memory_onboarding.py | |
| --ignore=ee/hogai/eval/ci/eval_ticket_summary.py | |
| --ignore=ee/hogai/eval/ci/eval_insight_search.py | |
| --ignore=ee/hogai/eval/ci/eval_ui_context.py | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| # Check out the actual branch instead of merge commit with master, | |
| # because we want the Braintrust experiment to have accurate git metadata (on master it's empty) | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| fetch-depth: 0 | |
| filter: blob:none | |
| clean: false | |
| - name: Clean up data directories with container permissions | |
| run: | | |
| # Use docker to clean up files created by containers | |
| [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true | |
| continue-on-error: true | |
| - name: Stop/Start stack with Docker Compose | |
| env: | |
| COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml | |
| run: | | |
| bin/ci-wait-for-docker launch --down | |
| - name: Wait for Docker services | |
| env: | |
| COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml | |
| run: bin/ci-wait-for-docker wait | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version-file: 'pyproject.toml' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0 | |
| with: | |
| version: '0.10.2' # pinned: unpinned setup-uv calls GH API on every job, exhausts rate limit | |
| enable-cache: true | |
| cache-dependency-glob: uv.lock | |
| save-cache: ${{ github.ref == 'refs/heads/master' }} | |
| - name: Install python dependencies | |
| shell: bash | |
| run: UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@0b1efabc08b657293548b77fb76cc02d26091c7e | |
| with: | |
| toolchain: 1.91.1 | |
| components: cargo | |
| - name: Cache Rust dependencies | |
| uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 | |
| with: | |
| shared-key: 'v2-rust-backend' | |
| workspaces: rust | |
| save-if: ${{ github.ref == 'refs/heads/master' }} | |
| - name: Install sqlx-cli | |
| run: cargo install sqlx-cli --version 0.8.0 --features postgres --no-default-features --locked | |
| - name: Add service hostnames to /etc/hosts | |
| run: sudo echo "127.0.0.1 db redis7 kafka clickhouse clickhouse-coordinator objectstorage seaweedfs temporal" | sudo tee -a /etc/hosts | |
| - name: Run LLM evals | |
| run: pytest ${{ matrix.test_paths }} ${{ matrix.ignore_paths }} -vv --junitxml=junit.xml | |
| env: | |
| EVAL_MODE: ci | |
| EXPORT_EVAL_RESULTS: true | |
| BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| INKEEP_API_KEY: ${{ secrets.INKEEP_API_KEY }} | |
| AZURE_INFERENCE_CREDENTIAL: ${{ secrets.AZURE_INFERENCE_CREDENTIAL }} | |
| AZURE_INFERENCE_ENDPOINT: ${{ secrets.AZURE_INFERENCE_ENDPOINT }} | |
| - name: Upload eval results | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 | |
| if: always() | |
| with: | |
| name: eval-results-${{ matrix.group }} | |
| path: | | |
| eval_results.jsonl | |
| junit.xml | |
| eval-summary: | |
| timeout-minutes: 5 | |
| name: AI evals / summary | |
| runs-on: ubuntu-latest | |
| needs: eval | |
| if: always() && needs.eval.result != 'skipped' | |
| permissions: | |
| contents: read | |
| issues: write | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: .github/scripts | |
| - name: Download all eval results | |
| uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 | |
| with: | |
| pattern: eval-results-* | |
| path: eval-artifacts | |
| - name: Merge eval results | |
| run: | | |
| # Merge all eval_results.jsonl files into one | |
| cat eval-artifacts/*/eval_results.jsonl > eval_results.jsonl 2>/dev/null || true | |
| if [ ! -s eval_results.jsonl ]; then | |
| echo "No eval results found" | |
| else | |
| echo "Merged eval results:" | |
| wc -l eval_results.jsonl | |
| fi | |
| - name: Post eval summary to PR | |
| # always() because we want to post even if some eval groups errored | |
| if: always() && github.event_name == 'pull_request' | |
| uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 | |
| with: | |
| script: | | |
| const fs = require("fs") | |
| const script = require('.github/scripts/post-eval-summary.js') | |
| script({ github, context, fs }) | |
| - name: Upload merged test results | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 | |
| if: always() | |
| with: | |
| name: junit-results-ai-evals | |
| path: eval-artifacts/*/junit.xml |