chore: remove upstream Slack CI notification workflow #8
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow runs all of our dagster tests. | |
| name: Dagster CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| env: | |
| SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only | |
| DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog' | |
| REDIS_URL: 'redis://localhost' | |
| CLICKHOUSE_HOST: 'localhost' | |
| CLICKHOUSE_SECURE: 'False' | |
| CLICKHOUSE_VERIFY: 'False' | |
| TEST: 1 | |
| OBJECT_STORAGE_ENABLED: 'True' | |
| OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000' | |
| OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user' | |
| OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password' | |
| # tests would intermittently fail in GH actions | |
| # with exit code 134 _after passing_ all tests | |
| # this appears to fix it | |
| # absolute wild tbh https://stackoverflow.com/a/75503402 | |
| DISPLAY: ':99.0' | |
| # this is a fake key so this workflow can run for external contributors as they do not have access to secrets (that we don't need here) | |
| OIDC_RSA_PRIVATE_KEY: ${{ vars.OIDC_RSA_FAKE_PRIVATE_KEY }} | |
| RUNS_ON_INTERNAL_PR: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false }} | |
| jobs: | |
| # Job to decide if we should run dagster ci | |
| # See https://github.com/dorny/paths-filter#conditional-execution for more details | |
| changes: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| name: Determine need to run dagster checks | |
| # Set job outputs to values from filter step | |
| outputs: | |
| dagster: ${{ steps.filter.outputs.dagster || 'true' }} | |
| oldest_supported: ${{ steps.read-versions.outputs.oldest_supported }} | |
| steps: | |
| # For pull requests it's not necessary to checkout the code, but we | |
| # also want this to run on master so we need to checkout | |
| - uses: actions/checkout@v6 | |
| with: | |
| clean: false | |
| - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 | |
| id: filter | |
| if: github.event_name != 'push' # Run all tests on master push | |
| with: | |
| filters: | | |
| dagster: | |
| - 'posthog/dags/**' | |
| # Dagster depends on Python, so ensure we run on any possible changes to it | |
| - 'posthog/**/*' | |
| - 'ee/**/*' | |
| - 'products/*' | |
| - 'products/*/!(frontend)/**' | |
| # Make sure we run if someone is explicitly change the workflow | |
| - .github/workflows/ci-dagster.yml | |
| - .github/clickhouse-versions.json | |
| # We use docker compose for tests, make sure we rerun on | |
| # changes to docker-compose.dev.yml e.g. dependency | |
| # version changes | |
| - docker-compose.dev.yml | |
| - frontend/public/email/* | |
| - name: Read ClickHouse versions from JSON | |
| id: read-versions | |
| if: github.event_name == 'push' || steps.filter.outputs.dagster == 'true' | |
| run: | | |
| oldest_supported=$(jq -r '.oldest_supported' .github/clickhouse-versions.json) | |
| if [ -z "$oldest_supported" ] || [ "$oldest_supported" = "null" ]; then | |
| echo "::error::No oldest_supported version found in .github/clickhouse-versions.json" | |
| exit 1 | |
| fi | |
| echo "oldest_supported=[\"$oldest_supported\"]" >> $GITHUB_OUTPUT | |
| dagster: | |
| name: Dagster tests | |
| needs: [changes] | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| clickhouse-server-image: ${{ fromJson(needs.changes.outputs.oldest_supported) }} | |
| if: needs.changes.outputs.dagster == 'true' | |
| runs-on: depot-ubuntu-latest | |
| steps: | |
| - name: 'Checkout repo' | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| clean: false | |
| - name: Clean up data directories with container permissions | |
| run: | | |
| # Use docker to clean up files created by containers | |
| [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true | |
| continue-on-error: true | |
| - name: Start stack with Docker Compose | |
| env: | |
| COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml | |
| CLICKHOUSE_SERVER_IMAGE: ${{ matrix.clickhouse-server-image }} | |
| run: | | |
| bin/ci-wait-for-docker launch --down | |
| - name: Wait for Docker services | |
| env: | |
| COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml | |
| CLICKHOUSE_SERVER_IMAGE: ${{ matrix.clickhouse-server-image }} | |
| run: bin/ci-wait-for-docker wait | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version-file: 'pyproject.toml' | |
| - name: Install uv | |
| id: setup-uv | |
| uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0 | |
| with: | |
| version: '0.10.2' # pinned: unpinned setup-uv calls GH API on every job, exhausts rate limit | |
| enable-cache: true | |
| cache-dependency-glob: uv.lock | |
| save-cache: ${{ github.ref == 'refs/heads/master' }} | |
| - name: Install SAML (python3-saml) dependencies | |
| if: steps.setup-uv.outputs.cache-hit != 'true' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl | |
| - name: Install python dependencies | |
| shell: bash | |
| run: | | |
| UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@0b1efabc08b657293548b77fb76cc02d26091c7e | |
| with: | |
| toolchain: stable | |
| components: cargo | |
| - name: Cache Rust dependencies | |
| uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 | |
| with: | |
| shared-key: 'v2-rust-backend' | |
| workspaces: rust | |
| save-if: ${{ github.ref == 'refs/heads/master' }} | |
| - name: Install sqlx-cli | |
| uses: ./.github/actions/setup-sqlx-cli | |
| - name: Add service hostnames to /etc/hosts | |
| run: sudo echo "127.0.0.1 db redis7 kafka clickhouse clickhouse-coordinator objectstorage seaweedfs temporal" | sudo tee -a /etc/hosts | |
| - name: Create Dagster test database | |
| run: | | |
| # Ensure the test_dagster database exists for Dagster's PostgreSQL-backed | |
| # event log / run storage (avoids SQLite locking issues in tests). | |
| # The init script in docker/postgres-init-scripts/ handles this on fresh | |
| # containers, but this step is a safety net. | |
| docker compose -f docker-compose.dev.yml exec -T db \ | |
| psql -U posthog -tAc "SELECT 1 FROM pg_database WHERE datname='test_dagster'" | grep -q 1 || \ | |
| docker compose -f docker-compose.dev.yml exec -T db \ | |
| psql -U posthog -c "CREATE DATABASE test_dagster;" | |
| - name: Run migrations | |
| run: | | |
| # Run Django migrations first | |
| python manage.py migrate | |
| # Then run persons migrations using sqlx | |
| DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \ | |
| sqlx database create | |
| DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \ | |
| sqlx migrate run --source rust/persons_migrations/ | |
| - name: Run clickhouse migrations | |
| run: | | |
| python manage.py migrate_clickhouse | |
| - name: Run Dagster tests | |
| run: | | |
| pytest posthog/dags --junitxml=junit-dagster.xml | |
| - name: Run products Dagster tests | |
| run: | | |
| pytest products/**/dags --junitxml=junit-products.xml | |
| - name: Upload test results | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 | |
| if: always() | |
| with: | |
| name: junit-results-dagster | |
| path: junit-*.xml | |
| # Job just to collate the status of the matrix jobs for requiring passing status | |
| dagster_tests: | |
| needs: [dagster] | |
| name: Dagster Tests Pass | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| if: always() | |
| steps: | |
| - name: Check matrix outcome | |
| run: | | |
| # The `needs.dagster.result` will be 'success' only if all jobs in the matrix succeeded. | |
| # Otherwise, it will be 'failure'. | |
| if [[ "${{ needs.dagster.result }}" != "success" && "${{ needs.dagster.result }}" != "skipped" ]]; then | |
| echo "One or more jobs in the Dagster test matrix failed." | |
| exit 1 | |
| fi | |
| echo "All checks passed." |