ci.yml

name: Tests And Linting

on:
  pull_request:
  push:
    branches:
      - main

# Cancel in-flight runs of this workflow on the same ref when a newer
# commit lands. Saves runner minutes during a churn of PR pushes. Not
# applied to publish.yml — releases must never be cancelled mid-upload.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  ci-scope:
    runs-on: ubuntu-latest
    outputs:
      ci_required: ${{ steps.scope.outputs.ci_required }}
    steps:
      - name: Check out repository
        uses: actions/checkout@v7
        with:
          fetch-depth: 0

      - name: Detect CI scope
        id: scope
        env:
          EVENT_NAME: ${{ github.event_name }}
          HEAD_SHA: ${{ github.sha }}
          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
          PUSH_BEFORE_SHA: ${{ github.event.before }}
        shell: bash
        run: |
          set -euo pipefail

          ci_required=true
          base_sha=""

          if [ "$EVENT_NAME" = "pull_request" ]; then
            base_sha="$PR_BASE_SHA"
          elif [ -n "${PUSH_BEFORE_SHA:-}" ] && [[ ! "$PUSH_BEFORE_SHA" =~ ^0+$ ]]; then
            base_sha="$PUSH_BEFORE_SHA"
          fi

          if [ -n "$base_sha" ] && git cat-file -e "$base_sha^{commit}" && git cat-file -e "$HEAD_SHA^{commit}"; then
            changed_files="$(git diff --name-only "$base_sha" "$HEAD_SHA")"
            if [ -n "$changed_files" ]; then
              ci_required=false
              while IFS= read -r path; do
                [ -z "$path" ] && continue
                case "$path" in
                  docs/examples|docs/examples/*)
                    ci_required=true
                    break
                    ;;
                  docs/*)
                    ;;
                  *.md)
                    case "$path" in
                      */*)
                        ci_required=true
                        break
                        ;;
                    esac
                    ;;
                  *)
                    ci_required=true
                    break
                    ;;
                esac
              done <<< "$changed_files"
            fi
          fi

          echo "ci_required=$ci_required" >> "$GITHUB_OUTPUT"
          echo "ci_required=$ci_required"

  quality:
    needs: [ci-scope]
    if: needs.ci-scope.outputs.ci_required == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v7

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: uv.lock

      - name: Set up Python
        run: uv python install 3.11

      - name: Cache uv-managed virtualenv
        uses: actions/cache@v6
        with:
          path: .venv
          key: venv-quality-${{ runner.os }}-py3.11-${{ hashFiles('uv.lock', 'pyproject.toml') }}
          restore-keys: |
            venv-quality-${{ runner.os }}-py3.11-

      - name: Install dependencies
        run: uv sync --all-extras --dev

      - name: Install Pre-Commit hooks
        run: uv run pre-commit install

      - name: Load cached Pre-Commit Dependencies
        id: cached-pre-commit-dependencies
        uses: actions/cache@v6
        with:
          path: ~/.cache/pre-commit/
          key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}

      # Each subsequent step uses `if: always()` so a failing earlier step
      # does not stop the others — a single failed run still reports every
      # tool's output, matching the parallel-jobs behavior of the old
      # validate/mypy/pyright/slotscheck split.
      - name: Execute Pre-Commit
        id: pre_commit
        run: uv run pre-commit run --show-diff-on-failure --color=always --all-files

      - name: Run mypy
        id: mypy
        if: always() && steps.pre_commit.conclusion != 'skipped' && steps.pre_commit.conclusion != 'cancelled'
        run: uv run mypy

      - name: Run pyright
        id: pyright
        if: always() && steps.mypy.conclusion != 'skipped' && steps.mypy.conclusion != 'cancelled'
        run: uv run pyright

      - name: Run slotscheck
        if: always() && steps.pyright.conclusion != 'skipped' && steps.pyright.conclusion != 'cancelled'
        run: uv run slotscheck sqlspec

  # test-unit: fast tier — no Docker, runs tests/unit only.
  # Acts as a gate for test-integration: a broken unit test cancels the
  # entire matrix (fail-fast: true) AND prevents integration jobs from
  # ever starting via `needs:` below.
  test-unit:
    needs: [ci-scope]
    if: needs.ci-scope.outputs.ci_required == 'true'
    runs-on: ubuntu-latest
    strategy:
      # fail-fast: true — matrix dimension is Python version only.
      # sqlspec failures are typically version-agnostic library bugs, so the
      # other 4 jobs running to completion after one fails wastes ~60 min of
      # billed runner time. test-build.yml gates package builds behind these
      # checks before spending runner minutes on wheel matrices.
      fail-fast: true
      matrix:
        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
    timeout-minutes: 15
    steps:
      - name: Check out repository
        uses: actions/checkout@v7

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: uv.lock

      - name: Set up Python
        run: uv python install ${{ matrix.python-version }}

      - name: Cache uv-managed virtualenv
        uses: actions/cache@v6
        with:
          path: .venv
          key: venv-unit-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('uv.lock', 'pyproject.toml') }}
          restore-keys: |
            venv-unit-${{ runner.os }}-py${{ matrix.python-version }}-

      - name: Install dependencies
        # tests/unit/ imports every adapter module, so the install set is
        # the same as integration. Speed comes from skipping Docker
        # pre-pull and running only tests/unit/, not from a lighter sync.
        run: uv sync --all-extras --dev

      - name: Test
        # covdefaults sets fail_under=100 globally; the per-tier run only
        # covers a slice of sqlspec, so override to 0 here. The combined
        # view in Codecov is the source of truth.
        run: >-
          uv run pytest tests/unit
          -n auto --dist=loadgroup
          --cov=sqlspec --cov-report=xml:coverage-unit.xml --cov-report=term-missing
          --cov-fail-under=0

      - name: Upload coverage to Codecov
        if: always()
        uses: codecov/codecov-action@v7
        with:
          files: coverage-unit.xml
          flags: unit,py${{ matrix.python-version }}
          name: test-unit-${{ matrix.python-version }}
          fail_ci_if_error: false
          token: ${{ secrets.CODECOV_TOKEN }}

  test-sqlglot-c:
    # The custom dialects extend sqlglot through TRANSFORMS/FACTOR patching and
    # parser method hooks whose constraints only surface under the compiled
    # sqlglotc overlay (sqlglot[c]). Run the dialect-adjacent unit tests with it.
    needs: [ci-scope]
    if: needs.ci-scope.outputs.ci_required == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Check out repository
        uses: actions/checkout@v7

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: uv.lock

      - name: Set up Python
        run: uv python install 3.13

      - name: Install dependencies
        run: uv sync --all-extras --dev

      - name: Test against compiled sqlglot
        run: >-
          uv run --with 'sqlglot[c]' pytest
          tests/unit/dialects tests/unit/builder
          -n auto --dist=loadgroup -p no:cacheprovider

  test-integration:
    needs: [ci-scope, test-unit]
    if: needs.ci-scope.outputs.ci_required == 'true'
    runs-on: ubuntu-24.04
    strategy:
      fail-fast: true
      matrix:
        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
    timeout-minutes: 30
    steps:
      - name: Check out repository
        uses: actions/checkout@v7

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: uv.lock

      - name: Set up Python
        run: uv python install ${{ matrix.python-version }}

      - name: Cache uv-managed virtualenv
        uses: actions/cache@v6
        with:
          path: .venv
          key: venv-integration-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('uv.lock', 'pyproject.toml') }}
          restore-keys: |
            venv-integration-${{ runner.os }}-py${{ matrix.python-version }}-

      - name: Install dependencies
        run: uv sync --all-extras --dev

      - name: Free runner disk before docker work
        run: |
          set -eu
          df -h /
          sudo rm -rf \
            /opt/az \
            /opt/ghc \
            /opt/hostedtoolcache/CodeQL \
            /opt/microsoft \
            /usr/lib/jvm \
            /usr/local/.ghcup \
            /usr/local/lib/android \
            /usr/local/lib/node_modules \
            /usr/local/share/boost \
            /usr/local/share/chromium \
            /usr/local/share/powershell \
            /usr/share/dotnet \
            /usr/share/miniconda \
            /usr/share/swift
          docker system prune -af || true
          sudo apt-get clean
          df -h /

      - name: Install SQL Server ODBC driver
        run: |
          set -eu
          key_url="https://packages.microsoft.com/keys/microsoft.asc"
          key_file="microsoft.asc"
          keyring_file="microsoft-prod.gpg"
          trap 'rm -f microsoft.asc microsoft-prod.gpg' EXIT

          for attempt in 1 2 3 4 5; do
            rm -f "${key_file}"
            if curl --fail --location --show-error --silent --output "${key_file}" "${key_url}" && gpg --show-keys "${key_file}" >/dev/null 2>&1; then
              break
            fi

            if [ "${attempt}" -eq 5 ]; then
              echo "Failed to download a valid Microsoft signing key" >&2
              ls -l "${key_file}" 2>/dev/null || true
              gpg --show-keys "${key_file}" || true
              exit 1
            fi

            echo "Microsoft signing key download was invalid; retrying (${attempt}/5)." >&2
            sleep $((attempt * 5))
          done

          gpg --dearmor --yes --output "${keyring_file}" "${key_file}"
          sudo install -d -m 0755 /usr/share/keyrings /etc/apt/sources.list.d
          sudo install -m 0644 "${keyring_file}" /usr/share/keyrings/microsoft-prod.gpg
          echo "deb [arch=amd64,arm64,armhf signed-by=/usr/share/keyrings/microsoft-prod.gpg] https://packages.microsoft.com/ubuntu/24.04/prod noble main" | sudo tee /etc/apt/sources.list.d/microsoft-prod.list >/dev/null
          sudo apt-get update
          sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 unixodbc-dev
          odbcinst -q -d -n "ODBC Driver 18 for SQL Server"

      - name: Cache docker images
        id: docker-image-cache
        uses: actions/cache@v6
        with:
          path: /tmp/docker-images
          key: docker-images-v2-${{ hashFiles('.github/workflows/ci.yml') }}

      - name: Load cached docker images
        if: steps.docker-image-cache.outputs.cache-hit == 'true'
        run: |
          docker load -i /tmp/docker-images/sqlspec-test-images.tar
          rm -f /tmp/docker-images/sqlspec-test-images.tar

      - name: Pre-pull docker images
        if: steps.docker-image-cache.outputs.cache-hit != 'true'
        run: |
          set -eu
          images=(
            postgres:18
            gizmodata/gizmosql:latest
            pgvector/pgvector:pg15
            paradedb/paradedb:0.21.5-pg16
            cockroachdb/cockroach:latest
            mysql:8.4
            mcr.microsoft.com/mssql/server:2022-latest
            gvenzl/oracle-free:23-slim-faststart
            ghcr.io/goccy/bigquery-emulator:latest
            gcr.io/cloud-spanner-emulator/emulator:latest
            rustfs/rustfs:latest
            rustfs/rc:latest
          )
          printf '%s\n' "${images[@]}" | xargs -P 4 -I{} docker pull {}
          mkdir -p /tmp/docker-images
          docker save "${images[@]}" -o /tmp/docker-images/sqlspec-test-images.tar

      - name: Test
        id: pytest
        env:
          PYTHONFAULTHANDLER: "1"
          PYTEST_ADDOPTS: "--max-worker-restart=0"
        # At 22min, broadcast SIGABRT to ALL python processes (controller + xdist
        # workers). Faulthandler in each process dumps every thread to stderr
        # before exit, so the test step log captures the actual hang site.
        run: |
          set -m
          (
            sleep 1320
            echo "::group::Hang detected - SIGABRT to all python"
            pkill -ABRT -f python || true
            sleep 10
            echo "::endgroup::"
          ) &
          aborter_pid=$!
          # covdefaults sets fail_under=100 globally; per-tier coverage
          # only covers a slice, so override to 0. Codecov merges the
          # tiers server-side.
          uv run pytest tests/integration docs/examples -n 2 --dist=loadgroup \
            --cov=sqlspec --cov-report=xml:coverage-integration.xml --cov-report=term-missing \
            --cov-fail-under=0
          rc=$?
          kill "$aborter_pid" 2>/dev/null || true
          exit $rc

      - name: Upload coverage to Codecov
        if: always() && steps.pytest.outcome != 'skipped'
        uses: codecov/codecov-action@v7
        with:
          files: coverage-integration.xml
          flags: integration,py${{ matrix.python-version }}
          name: test-integration-${{ matrix.python-version }}
          fail_ci_if_error: false
          token: ${{ secrets.CODECOV_TOKEN }}

      - name: Dump python stacks on hang
        if: failure() && steps.pytest.outcome == 'failure'
        run: |
          pip install --quiet py-spy || true
          for pid in $(pgrep -f 'pytest|python' || true); do
            echo "=== py-spy dump pid=$pid ==="
            sudo env "PATH=$PATH" py-spy dump --pid "$pid" || true
          done
          echo "=== docker ps ==="
          docker ps -a
          echo "=== docker logs (last 100 lines each) ==="
          for c in $(docker ps -aq); do
            echo "--- container $c ---"
            docker logs --tail 100 "$c" 2>&1 || true
          done

  # test-linux-freethreaded:
  #   # Disabled: cp314t ecosystem not mature enough yet (missing binary wheels, long build times)
  #   runs-on: ubuntu-latest
  #   continue-on-error: true
  #   timeout-minutes: 30
  #   steps:
  #     - uses: actions/checkout@v4
  #     - uses: astral-sh/setup-uv@v3
  #     - run: uv python install 3.14t
  #     - run: uv sync --extra aiosqlite --extra duckdb --dev
  #     - run: uv run pytest tests/unit -n 2 --dist=loadgroup

  # test-windows:
  #   runs-on: windows-latest
  #   strategy:
  #     fail-fast: true
  #     matrix:
  #       python-version: ["3.12", "3.13"]
  #   timeout-minutes: 30
  #   steps:
  #     - name: Check out repository
  #       uses: actions/checkout@v4

  #     - name: Install uv
  #       uses: astral-sh/setup-uv@v3

  #     - name: Set up Python
  #       run: uv python install ${{ matrix.python-version }}

  #     - name: Install dependencies
  #       run: uv sync --all-extras --dev

  #     - name: Test
  #       run: uv run pytest -m ""

  # test-osx:
  #   runs-on: macos-latest
  #   strategy:
  #     fail-fast: true
  #     matrix:
  #       python-version: ["3.11", "3.12", "3.13"]
  #   timeout-minutes: 30
  #   steps:
  #     - name: Check out repository
  #       uses: actions/checkout@v4

  #     - name: Install uv
  #       uses: astral-sh/setup-uv@v3

  #     - name: Set up Python
  #       run: uv python install ${{ matrix.python-version }}

  #     - name: Install dependencies
  #       run: uv sync --all-extras --dev

  #     - name: Test
  #       run: uv run pytest -m ""

  build-docs:
    needs:
      - ci-scope
      - quality
      - test-unit
    if: github.event_name == 'pull_request' && needs.ci-scope.outputs.ci_required == 'true'
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository
        uses: actions/checkout@v7

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: uv.lock

      - name: Set up Python
        run: uv python install 3.12

      - name: Install dependencies
        run: uv sync --all-extras --dev

      - name: Build docs
        run: uv run make docs

  ci-required:
    needs:
      - ci-scope
      - quality
      - test-unit
      - test-integration
      - build-docs
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Verify required CI result
        env:
          SCOPE_RESULT: ${{ needs.ci-scope.result }}
          CI_REQUIRED: ${{ needs.ci-scope.outputs.ci_required }}
          QUALITY_RESULT: ${{ needs.quality.result }}
          UNIT_RESULT: ${{ needs.test-unit.result }}
          INTEGRATION_RESULT: ${{ needs.test-integration.result }}
          DOCS_RESULT: ${{ needs.build-docs.result }}
          EVENT_NAME: ${{ github.event_name }}
        shell: bash
        run: |
          set -euo pipefail

          if [ "$SCOPE_RESULT" != "success" ]; then
            echo "::error::SCOPE_RESULT was $SCOPE_RESULT"
            exit 1
          fi

          if [ "$CI_REQUIRED" = "false" ]; then
            echo "Heavy CI intentionally skipped for docs-only or root markdown changes."
            exit 0
          fi

          if [ "$CI_REQUIRED" != "true" ]; then
            echo "::error::CI_REQUIRED was '$CI_REQUIRED'"
            exit 1
          fi

          failed=0

          for check in QUALITY_RESULT UNIT_RESULT INTEGRATION_RESULT; do
            result="${!check}"
            if [ "$result" != "success" ]; then
              echo "::error::$check was $result"
              failed=1
            fi
          done

          if [ "$EVENT_NAME" = "pull_request" ] && [ "$DOCS_RESULT" != "success" ]; then
            echo "::error::DOCS_RESULT was $DOCS_RESULT"
            failed=1
          fi

          exit "$failed"