ci-dagster.yml

# This workflow runs all of our dagster tests.
name: Dagster CI
on:
    push:
        branches:
            - master
    pull_request:

permissions:
    contents: read
    pull-requests: read

env:
    SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only
    DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
    REDIS_URL: 'redis://localhost'
    CLICKHOUSE_HOST: 'localhost'
    CLICKHOUSE_SECURE: 'False'
    CLICKHOUSE_VERIFY: 'False'
    TEST: 1
    OBJECT_STORAGE_ENABLED: 'True'
    OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000'
    OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
    OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
    # tests would intermittently fail in GH actions
    # with exit code 134 _after passing_ all tests
    # this appears to fix it
    # absolute wild tbh https://stackoverflow.com/a/75503402
    DISPLAY: ':99.0'
    # this is a fake key so this workflow can run for external contributors as they do not have access to secrets (that we don't need here)
    OIDC_RSA_PRIVATE_KEY: ${{ vars.OIDC_RSA_FAKE_PRIVATE_KEY }}
    RUNS_ON_INTERNAL_PR: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false }}

jobs:
    # Job to decide if we should run dagster ci
    # See https://github.com/dorny/paths-filter#conditional-execution for more details
    changes:
        runs-on: ubuntu-latest
        timeout-minutes: 5
        name: Determine need to run dagster checks
        # Set job outputs to values from filter step
        outputs:
            dagster: ${{ steps.filter.outputs.dagster || 'true' }}
            oldest_supported: ${{ steps.read-versions.outputs.oldest_supported }}
        steps:
            # For pull requests it's not necessary to checkout the code, but we
            # also want this to run on master so we need to checkout
            - uses: actions/checkout@v6
              with:
                  clean: false

            - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
              id: filter
              if: github.event_name != 'push' # Run all tests on master push
              with:
                  filters: |
                      dagster:
                        - 'posthog/dags/**'
                        # Dagster depends on Python, so ensure we run on any possible changes to it
                        - 'posthog/**/*'
                        - 'ee/**/*'
                        - 'products/*'
                        - 'products/*/!(frontend)/**'
                        # Make sure we run if someone is explicitly change the workflow
                        - .github/workflows/ci-dagster.yml
                        - .github/clickhouse-versions.json
                        # We use docker compose for tests, make sure we rerun on
                        # changes to docker-compose.dev.yml e.g. dependency
                        # version changes
                        - docker-compose.dev.yml
                        - frontend/public/email/*

            - name: Read ClickHouse versions from JSON
              id: read-versions
              if: github.event_name == 'push' || steps.filter.outputs.dagster == 'true'
              run: |
                  oldest_supported=$(jq -r '.oldest_supported' .github/clickhouse-versions.json)
                  if [ -z "$oldest_supported" ] || [ "$oldest_supported" = "null" ]; then
                    echo "::error::No oldest_supported version found in .github/clickhouse-versions.json"
                    exit 1
                  fi
                  echo "oldest_supported=[\"$oldest_supported\"]" >> $GITHUB_OUTPUT

    dagster:
        name: Dagster tests
        needs: [changes]
        timeout-minutes: 30
        strategy:
            fail-fast: false
            matrix:
                clickhouse-server-image: ${{ fromJson(needs.changes.outputs.oldest_supported) }}
        if: needs.changes.outputs.dagster == 'true'
        runs-on: depot-ubuntu-latest
        steps:
            - name: 'Checkout repo'
              uses: actions/checkout@v6
              with:
                  fetch-depth: 1
                  clean: false
            - name: Clean up data directories with container permissions
              run: |
                  # Use docker to clean up files created by containers
                  [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
              continue-on-error: true

            - name: Start stack with Docker Compose
              env:
                  COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml
                  CLICKHOUSE_SERVER_IMAGE: ${{ matrix.clickhouse-server-image }}
              run: |
                  bin/ci-wait-for-docker launch --down

            - name: Wait for Docker services
              env:
                  COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml
                  CLICKHOUSE_SERVER_IMAGE: ${{ matrix.clickhouse-server-image }}
              run: bin/ci-wait-for-docker wait

            - name: Set up Python
              uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
              with:
                  python-version-file: 'pyproject.toml'

            - name: Install uv
              id: setup-uv
              uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0
              with:
                  version: '0.10.2' # pinned: unpinned setup-uv calls GH API on every job, exhausts rate limit
                  enable-cache: true
                  cache-dependency-glob: uv.lock
                  save-cache: ${{ github.ref == 'refs/heads/master' }}

            - name: Install SAML (python3-saml) dependencies
              if: steps.setup-uv.outputs.cache-hit != 'true'
              run: |
                  sudo apt-get update
                  sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl

            - name: Install python dependencies
              shell: bash
              run: |
                  UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev

            - name: Install Rust
              uses: dtolnay/rust-toolchain@0b1efabc08b657293548b77fb76cc02d26091c7e
              with:
                  toolchain: stable
                  components: cargo

            - name: Cache Rust dependencies
              uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2
              with:
                  shared-key: 'v2-rust-backend'
                  workspaces: rust
                  save-if: ${{ github.ref == 'refs/heads/master' }}

            - name: Install sqlx-cli
              uses: ./.github/actions/setup-sqlx-cli

            - name: Add service hostnames to /etc/hosts
              run: sudo echo "127.0.0.1 db redis7 kafka clickhouse clickhouse-coordinator objectstorage seaweedfs temporal" | sudo tee -a /etc/hosts

            - name: Create Dagster test database
              run: |
                  # Ensure the test_dagster database exists for Dagster's PostgreSQL-backed
                  # event log / run storage (avoids SQLite locking issues in tests).
                  # The init script in docker/postgres-init-scripts/ handles this on fresh
                  # containers, but this step is a safety net.
                  docker compose -f docker-compose.dev.yml exec -T db \
                    psql -U posthog -tAc "SELECT 1 FROM pg_database WHERE datname='test_dagster'" | grep -q 1 || \
                    docker compose -f docker-compose.dev.yml exec -T db \
                      psql -U posthog -c "CREATE DATABASE test_dagster;"

            - name: Run migrations
              run: |
                  # Run Django migrations first
                  python manage.py migrate
                  # Then run persons migrations using sqlx
                  DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
                    sqlx database create
                  DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
                    sqlx migrate run --source rust/persons_migrations/

            - name: Run clickhouse migrations
              run: |
                  python manage.py migrate_clickhouse

            - name: Run Dagster tests
              run: |
                  pytest posthog/dags --junitxml=junit-dagster.xml

            - name: Run products Dagster tests
              run: |
                  pytest products/**/dags --junitxml=junit-products.xml

            - name: Upload test results
              uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
              if: always()
              with:
                  name: junit-results-dagster
                  path: junit-*.xml

    # Job just to collate the status of the matrix jobs for requiring passing status
    dagster_tests:
        needs: [dagster]
        name: Dagster Tests Pass
        runs-on: ubuntu-latest
        timeout-minutes: 5
        if: always()
        steps:
            - name: Check matrix outcome
              run: |
                  # The `needs.dagster.result` will be 'success' only if all jobs in the matrix succeeded.
                  # Otherwise, it will be 'failure'.
                  if [[ "${{ needs.dagster.result }}" != "success" && "${{ needs.dagster.result }}" != "skipped" ]]; then
                    echo "One or more jobs in the Dagster test matrix failed."
                    exit 1
                  fi

                  echo "All checks passed."