Nightly snapshot recording #126
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Nightly snapshot recording | |
| on: | |
| schedule: | |
| - cron: '0 2 * * *' # 2 AM UTC daily | |
| workflow_dispatch: | |
| inputs: | |
| dataSource: | |
| description: Record snapshots for this data source | |
| type: choice | |
| default: all | |
| options: | |
| - all | |
| - athena | |
| - bigquery | |
| - databricks | |
| - fabric | |
| - postgres | |
| - redshift | |
| - snowflake | |
| - sqlserver | |
| - synapse | |
| - trino-postgres | |
| - trino-s3 | |
| env: | |
| PROJECT_NAME: soda-core | |
| jobs: | |
| # ───────────────────────────────────────────────────────────────────── | |
| # 1. Define the test matrix (reuse existing script) | |
| # ───────────────────────────────────────────────────────────────────── | |
| define-test-matrix: | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| modules: ${{ steps.modules.outputs.modules }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Resolve data source selection | |
| env: | |
| INPUT_DATASOURCE: ${{ inputs.dataSource }} | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| DS="$INPUT_DATASOURCE" | |
| [ -z "$DS" ] && DS="all" | |
| else | |
| DS="all" | |
| fi | |
| echo "DATA_SOURCE=$DS" >> "$GITHUB_ENV" | |
| - name: Define modules | |
| id: modules | |
| run: | | |
| echo "INFO: DATA_SOURCE is set to ${DATA_SOURCE}" | |
| if [ "${DATA_SOURCE}" = "all" ]; then | |
| echo modules=$(bash scripts/test_matrix.sh) >> "$GITHUB_OUTPUT" | |
| else | |
| echo 'modules=["__DATA_SOURCE__"]' | sed "s|__DATA_SOURCE__|${DATA_SOURCE}|g" >> "$GITHUB_OUTPUT" | |
| fi | |
| # ───────────────────────────────────────────────────────────────────── | |
| # 2. Record snapshots for each data source | |
| # ───────────────────────────────────────────────────────────────────── | |
| record-snapshots: | |
| runs-on: ubuntu-24.04 | |
| needs: [define-test-matrix] | |
| services: | |
| postgres: | |
| image: ${{ ( matrix.module == 'postgres' || startsWith(matrix.module, 'trino') ) && 'postgres:15.10-alpine3.21' || '' }} | |
| env: | |
| POSTGRES_USER: soda_test | |
| POSTGRES_DB: soda_test | |
| POSTGRES_HOST_AUTH_METHOD: trust | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| --health-start-period 10s | |
| ports: | |
| - 5432:5432 | |
| sqlserver: | |
| image: ${{ ( matrix.module == 'sqlserver' ) && 'mcr.microsoft.com/mssql/server:2022-latest' || '' }} | |
| env: | |
| ACCEPT_EULA: Y | |
| SA_PASSWORD: Password1! | |
| ports: | |
| - 1433:1433 | |
| options: >- | |
| --health-cmd "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P Password1! -Q 'select 1' -C -b -o /dev/null" | |
| --health-interval 1s | |
| --health-timeout 2s | |
| --health-retries 10 | |
| --health-start-period 10s | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| module: ${{ fromJSON(needs.define-test-matrix.outputs.modules) }} | |
| # duckdb and sparkdf run fast enough without snapshots | |
| exclude: | |
| - module: duckdb | |
| - module: sparkdf | |
| env: | |
| SNOWFLAKE_ACCOUNT: ${{ vars.SNOWFLAKE_CI_ACCOUNT }} | |
| SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_CI_USERNAME }} | |
| SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_CI_DATABASE }} | |
| DATABRICKS_HOST: ${{ vars.DATABRICKS_CI_HOST }} | |
| DATABRICKS_HTTP_PATH: ${{ vars.DATABRICKS_CI_HTTP_PATH }} | |
| DATABRICKS_CATALOG: ${{ vars.DATABRICKS_CI_CATALOG }} | |
| REDSHIFT_HOST: ${{ vars.REDSHIFT_CI_HOST }} | |
| REDSHIFT_USERNAME: ${{ vars.REDSHIFT_CI_USERNAME }} | |
| REDSHIFT_DATABASE: "soda_test" | |
| REDSHIFT_PORT: "5439" | |
| ATHENA_S3_TEST_DIR: ${{ vars.ATHENA_CI_STAGING_DIR }} | |
| ATHENA_SCHEMA: ${{ vars.ATHENA_CI_SCHEMA }} | |
| ATHENA_WORKGROUP: ${{ vars.ATHENA_CI_WORKGROUP }} | |
| SQLSERVER_USERNAME: sa | |
| SQLSERVER_PASSWORD: Password1! | |
| SQLSERVER_DATABASE: master | |
| SQLSERVER_SCHEMA: dbo | |
| SYNAPSE_HOST: ${{ vars.MICROSOFT_SYNAPSE_CI_HOST }} | |
| SYNAPSE_DATABASE: sodacisynapse | |
| SYNAPSE_AUTHENTICATION_TYPE: activedirectoryserviceprincipal | |
| FABRIC_HOST: ${{ vars.MICROSOFT_FABRIC_CI_HOST }} | |
| FABRIC_DATABASE: soda-ci-fabric-warehouse | |
| FABRIC_AUTHENTICATION_TYPE: activedirectoryserviceprincipal | |
| SODA_CORE_TELEMETRY_LOCAL_TEST_MODE: "true" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Resolve module | |
| run: | | |
| MODULE="${{ matrix.module }}" | |
| case "$MODULE" in | |
| trino-postgres) echo "TEST_MODULE=trino" >> "$GITHUB_ENV"; echo "TRINO_CATALOG=db" >> "$GITHUB_ENV" ;; | |
| trino-s3) echo "TEST_MODULE=trino" >> "$GITHUB_ENV"; echo "TRINO_CATALOG=iceberg" >> "$GITHUB_ENV" ;; | |
| *) echo "TEST_MODULE=$MODULE" >> "$GITHUB_ENV" ;; | |
| esac | |
| # CACHE_KEY uses the matrix module name (e.g. "trino-postgres") to avoid | |
| # collisions between variants that share the same TEST_MODULE (e.g. both | |
| # trino-postgres and trino-s3 resolve to TEST_MODULE=trino). | |
| echo "CACHE_KEY=$MODULE" >> "$GITHUB_ENV" | |
| - name: Set up Python 3.14 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.14" | |
| - name: Set up UV | |
| uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 | |
| - name: Install dependencies | |
| run: | | |
| curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - | |
| curl https://packages.microsoft.com/config/ubuntu/21.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list > /dev/null | |
| sudo apt-get update | |
| ACCEPT_EULA=Y sudo apt-get install -y libsasl2-dev msodbcsql18 | |
| - name: Get external secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802 # v2.0.10 | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_BUILD_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_BUILD_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_BUILD_DEFAULT_REGION }} | |
| with: | |
| secret-ids: | | |
| ,/soda/github/common/data-sources/envs | |
| parse-json-secrets: true | |
| - name: Get external secrets (special) | |
| uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802 # v2.0.10 | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_BUILD_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_BUILD_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_BUILD_DEFAULT_REGION }} | |
| with: | |
| secret-ids: | | |
| BIGQUERY_ACCOUNT_INFO_JSON,/soda/github/common/data-sources/BIGQUERY_ACCOUNT_INFO_JSON | |
| - name: Configure Trino | |
| if: startsWith(matrix.module, 'trino') | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_BUILD_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_BUILD_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_BUILD_DEFAULT_REGION }} | |
| TRINO_CI_ASSUME_ROLE_ARN: ${{ secrets.TRINO_CI_ASSUME_ROLE_ARN }} | |
| CI_MODULE: ${{ matrix.module }} | |
| run: bash scripts/start_trino_ci.sh | |
| - name: Sync packages | |
| run: uv sync --locked --all-packages --group dev | |
| # ── Record integration test snapshots ─────────────────────────── | |
| - name: Record integration test snapshots | |
| env: | |
| SODA_TEST_SNAPSHOT: record | |
| TEST_DATASOURCE: ${{ env.TEST_MODULE }} | |
| run: uv run python -m pytest -ra soda-tests/tests/integration | |
| # ── Run no_snapshot tests against real DB ─────────────────────── | |
| - name: Run no_snapshot tests | |
| env: | |
| TEST_DATASOURCE: ${{ env.TEST_MODULE }} | |
| run: | | |
| # Run tests that are excluded from snapshot mode against the real DB. | |
| # Use -m to select only no_snapshot-marked tests; exit 0 if none found. | |
| uv run python -m pytest -ra soda-tests/tests/integration -m "no_snapshot" \ | |
| --no-header -q || { | |
| rc=$? | |
| # Exit code 5 = "no tests collected" — that's fine, not all DS have no_snapshot tests | |
| [ $rc -eq 5 ] && exit 0 | |
| exit $rc | |
| } | |
| # ── Run nightly-only tests against real DB ─────────────────── | |
| - name: Run nightly-only tests | |
| env: | |
| SODA_NIGHTLY: "true" | |
| TEST_DATASOURCE: ${{ env.TEST_MODULE }} | |
| run: | | |
| uv run python -m pytest -ra soda-tests/tests/integration -m "nightly_only" \ | |
| --no-header -q || { | |
| rc=$? | |
| [ $rc -eq 5 ] && exit 0 | |
| exit $rc | |
| } | |
| # ── Verify recorded snapshots replay correctly ──────────────── | |
| - name: Verify replay | |
| env: | |
| SODA_TEST_SNAPSHOT: replay | |
| SODA_TEST_SNAPSHOT_FALLBACK: "false" | |
| TEST_DATASOURCE: ${{ env.TEST_MODULE }} | |
| run: uv run python -m pytest -ra soda-tests/tests/integration | |
| # ── Save snapshots to cache ──────────────────────────────────── | |
| - name: Save snapshot cache | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: .test_snapshots/ | |
| key: snapshots-${{ env.CACHE_KEY }}-${{ github.run_id }} | |
| - name: Dump Trino logs | |
| if: always() && startsWith(matrix.module, 'trino') | |
| run: | | |
| echo "=== Trino container logs ===" | |
| docker logs trino-ci 2>&1 | tail -100 | |
| - name: Stop Trino container | |
| if: always() && startsWith(matrix.module, 'trino') | |
| run: docker rm -f trino-ci 2>/dev/null || true | |