nightly-snapshot-record.yaml

---
name: Nightly snapshot recording

on:
  schedule:
    - cron: '0 2 * * *'  # 2 AM UTC daily
  workflow_dispatch:
    inputs:
      dataSource:
        description: Record snapshots for this data source
        type: choice
        default: all
        options:
          - all
          - athena
          - bigquery
          - databricks
          - fabric
          - postgres
          - redshift
          - snowflake
          - sqlserver
          - synapse
          - trino-postgres
          - trino-s3

env:
  PROJECT_NAME: soda-core

jobs:
  # ─────────────────────────────────────────────────────────────────────
  # 1. Define the test matrix (reuse existing script)
  # ─────────────────────────────────────────────────────────────────────
  define-test-matrix:
    runs-on: ubuntu-24.04
    outputs:
      modules: ${{ steps.modules.outputs.modules }}
    steps:
      - uses: actions/checkout@v4
      - name: Resolve data source selection
        env:
          INPUT_DATASOURCE: ${{ inputs.dataSource }}
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            DS="$INPUT_DATASOURCE"
            [ -z "$DS" ] && DS="all"
          else
            DS="all"
          fi
          echo "DATA_SOURCE=$DS" >> "$GITHUB_ENV"
      - name: Define modules
        id: modules
        run: |
          echo "INFO: DATA_SOURCE is set to ${DATA_SOURCE}"
          if [ "${DATA_SOURCE}" = "all" ]; then
            echo modules=$(bash scripts/test_matrix.sh) >> "$GITHUB_OUTPUT"
          else
            echo 'modules=["__DATA_SOURCE__"]' | sed "s|__DATA_SOURCE__|${DATA_SOURCE}|g" >> "$GITHUB_OUTPUT"
          fi

  # ─────────────────────────────────────────────────────────────────────
  # 2. Record snapshots for each data source
  # ─────────────────────────────────────────────────────────────────────
  record-snapshots:
    runs-on: ubuntu-24.04
    needs: [define-test-matrix]
    services:
      postgres:
        image: ${{ ( matrix.module == 'postgres' || startsWith(matrix.module, 'trino') ) && 'postgres:15.10-alpine3.21' || '' }}
        env:
          POSTGRES_USER: soda_test
          POSTGRES_DB: soda_test
          POSTGRES_HOST_AUTH_METHOD: trust
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
          --health-start-period 10s
        ports:
          - 5432:5432
      sqlserver:
        image: ${{ ( matrix.module == 'sqlserver' ) && 'mcr.microsoft.com/mssql/server:2022-latest' || '' }}
        env:
          ACCEPT_EULA: Y
          SA_PASSWORD: Password1!
        ports:
          - 1433:1433
        options: >-
          --health-cmd "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P Password1! -Q 'select 1' -C -b -o /dev/null"
          --health-interval 1s
          --health-timeout 2s
          --health-retries 10
          --health-start-period 10s
    strategy:
      fail-fast: false
      matrix:
        module: ${{ fromJSON(needs.define-test-matrix.outputs.modules) }}
        # duckdb and sparkdf run fast enough without snapshots
        exclude:
          - module: duckdb
          - module: sparkdf
    env:
      SNOWFLAKE_ACCOUNT: ${{ vars.SNOWFLAKE_CI_ACCOUNT }}
      SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_CI_USERNAME }}
      SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_CI_DATABASE }}
      DATABRICKS_HOST: ${{ vars.DATABRICKS_CI_HOST }}
      DATABRICKS_HTTP_PATH: ${{ vars.DATABRICKS_CI_HTTP_PATH }}
      DATABRICKS_CATALOG: ${{ vars.DATABRICKS_CI_CATALOG }}
      REDSHIFT_HOST: ${{ vars.REDSHIFT_CI_HOST }}
      REDSHIFT_USERNAME: ${{ vars.REDSHIFT_CI_USERNAME }}
      REDSHIFT_DATABASE: "soda_test"
      REDSHIFT_PORT: "5439"
      ATHENA_S3_TEST_DIR: ${{ vars.ATHENA_CI_STAGING_DIR }}
      ATHENA_SCHEMA: ${{ vars.ATHENA_CI_SCHEMA }}
      ATHENA_WORKGROUP: ${{ vars.ATHENA_CI_WORKGROUP }}
      SQLSERVER_USERNAME: sa
      SQLSERVER_PASSWORD: Password1!
      SQLSERVER_DATABASE: master
      SQLSERVER_SCHEMA: dbo
      SYNAPSE_HOST: ${{ vars.MICROSOFT_SYNAPSE_CI_HOST }}
      SYNAPSE_DATABASE: sodacisynapse
      SYNAPSE_AUTHENTICATION_TYPE: activedirectoryserviceprincipal
      FABRIC_HOST: ${{ vars.MICROSOFT_FABRIC_CI_HOST }}
      FABRIC_DATABASE: soda-ci-fabric-warehouse
      FABRIC_AUTHENTICATION_TYPE: activedirectoryserviceprincipal
      SODA_CORE_TELEMETRY_LOCAL_TEST_MODE: "true"

    steps:
      - uses: actions/checkout@v4
      - name: Resolve module
        run: |
          MODULE="${{ matrix.module }}"
          case "$MODULE" in
            trino-postgres) echo "TEST_MODULE=trino" >> "$GITHUB_ENV"; echo "TRINO_CATALOG=db" >> "$GITHUB_ENV" ;;
            trino-s3)       echo "TEST_MODULE=trino" >> "$GITHUB_ENV"; echo "TRINO_CATALOG=iceberg" >> "$GITHUB_ENV" ;;
            *)              echo "TEST_MODULE=$MODULE" >> "$GITHUB_ENV" ;;
          esac
          # CACHE_KEY uses the matrix module name (e.g. "trino-postgres") to avoid
          # collisions between variants that share the same TEST_MODULE (e.g. both
          # trino-postgres and trino-s3 resolve to TEST_MODULE=trino).
          echo "CACHE_KEY=$MODULE" >> "$GITHUB_ENV"
      - name: Set up Python 3.14
        uses: actions/setup-python@v5
        with:
          python-version: "3.14"
      - name: Set up UV
        uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081  # v5

      - name: Install dependencies
        run: |
          curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
          curl https://packages.microsoft.com/config/ubuntu/21.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list > /dev/null
          sudo apt-get update
          ACCEPT_EULA=Y sudo apt-get install -y libsasl2-dev msodbcsql18

      - name: Get external secrets
        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802  # v2.0.10
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_BUILD_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_BUILD_SECRET_ACCESS_KEY }}
          AWS_REGION: ${{ secrets.AWS_BUILD_DEFAULT_REGION }}
        with:
          secret-ids: |
            ,/soda/github/common/data-sources/envs
          parse-json-secrets: true

      - name: Get external secrets (special)
        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802  # v2.0.10
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_BUILD_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_BUILD_SECRET_ACCESS_KEY }}
          AWS_REGION: ${{ secrets.AWS_BUILD_DEFAULT_REGION }}
        with:
          secret-ids: |
            BIGQUERY_ACCOUNT_INFO_JSON,/soda/github/common/data-sources/BIGQUERY_ACCOUNT_INFO_JSON

      - name: Configure Trino
        if: startsWith(matrix.module, 'trino')
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_BUILD_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_BUILD_SECRET_ACCESS_KEY }}
          AWS_REGION: ${{ secrets.AWS_BUILD_DEFAULT_REGION }}
          TRINO_CI_ASSUME_ROLE_ARN: ${{ secrets.TRINO_CI_ASSUME_ROLE_ARN }}
          CI_MODULE: ${{ matrix.module }}
        run: bash scripts/start_trino_ci.sh

      - name: Sync packages
        run: uv sync --locked --all-packages --group dev

      # ── Record integration test snapshots ───────────────────────────
      - name: Record integration test snapshots
        env:
          SODA_TEST_SNAPSHOT: record
          TEST_DATASOURCE: ${{ env.TEST_MODULE }}
        run: uv run python -m pytest -ra soda-tests/tests/integration

      # ── Run no_snapshot tests against real DB ───────────────────────
      - name: Run no_snapshot tests
        env:
          TEST_DATASOURCE: ${{ env.TEST_MODULE }}
        run: |
          # Run tests that are excluded from snapshot mode against the real DB.
          # Use -m to select only no_snapshot-marked tests; exit 0 if none found.
          uv run python -m pytest -ra soda-tests/tests/integration -m "no_snapshot" \
            --no-header -q || {
            rc=$?
            # Exit code 5 = "no tests collected" — that's fine, not all DS have no_snapshot tests
            [ $rc -eq 5 ] && exit 0
            exit $rc
          }

      # ── Run nightly-only tests against real DB ───────────────────
      - name: Run nightly-only tests
        env:
          SODA_NIGHTLY: "true"
          TEST_DATASOURCE: ${{ env.TEST_MODULE }}
        run: |
          uv run python -m pytest -ra soda-tests/tests/integration -m "nightly_only" \
            --no-header -q || {
            rc=$?
            [ $rc -eq 5 ] && exit 0
            exit $rc
          }

      # ── Verify recorded snapshots replay correctly ────────────────
      - name: Verify replay
        env:
          SODA_TEST_SNAPSHOT: replay
          SODA_TEST_SNAPSHOT_FALLBACK: "false"
          TEST_DATASOURCE: ${{ env.TEST_MODULE }}
        run: uv run python -m pytest -ra soda-tests/tests/integration

      # ── Save snapshots to cache ────────────────────────────────────
      - name: Save snapshot cache
        uses: actions/cache/save@v4
        with:
          path: .test_snapshots/
          key: snapshots-${{ env.CACHE_KEY }}-${{ github.run_id }}

      - name: Dump Trino logs
        if: always() && startsWith(matrix.module, 'trino')
        run: |
          echo "=== Trino container logs ==="
          docker logs trino-ci 2>&1 | tail -100
      - name: Stop Trino container
        if: always() && startsWith(matrix.module, 'trino')
        run: docker rm -f trino-ci 2>/dev/null || true