test-build.yml

name: Test Build Configuration

on:
  workflow_dispatch:
    inputs:
      test_matrix:
        description: 'Test full matrix or subset'
        required: false
        default: 'subset'
        type: choice
        options:
          - 'full'
          - 'subset'
      python_version:
        description: 'Python version to build/test (single value; default 3.12)'
        required: false
        default: '3.12'
        type: choice
        options:
          - '3.10'
          - '3.11'
          - '3.12'
          - '3.13'
          - '3.14'
  pull_request:
    paths:
      - '.github/workflows/test-build.yml'
      - '.github/workflows/pgo-validate.yml'
      - '.github/workflows/publish.yml'
      - 'pyproject.toml'
      - 'uv.lock'
      - 'tools/scripts/pgo_training.py'
      - 'sqlspec/**'
      - 'tools/scripts/mypyc_smoke.py'

permissions:
  contents: read
  checks: read

# Cancel in-flight runs of this workflow on the same ref when a newer
# commit lands. Saves runner minutes during a churn of PR pushes. Not
# applied to publish.yml — releases must never be cancelled mid-upload.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  detect-mode:
    name: Detect matrix mode
    runs-on: ubuntu-latest
    outputs:
      matrix_mode: ${{ steps.detect.outputs.matrix_mode }}
    steps:
      - name: Check out repository
        uses: actions/checkout@v7
        with:
          fetch-depth: 0

      # workflow_dispatch with test_matrix=full always wins. Otherwise,
      # promote a PR to the full per-OS x per-py matrix when its diff
      # touches anything that could affect wheel-build correctness
      # (workflow yaml, packaging metadata, PGO training, smoke script).
      # Non-infra PRs stay in subset mode (linux x py3.12) for fast feedback.
      - name: Detect matrix mode
        id: detect
        env:
          DISPATCH_MODE: ${{ github.event.inputs.test_matrix }}
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        shell: bash
        run: |
          if [ "$DISPATCH_MODE" = "full" ]; then
            mode=full
            reason="workflow_dispatch input"
          else
            mode=subset
            reason="default"
            if [ "$GITHUB_EVENT_NAME" = "pull_request" ] && [ -n "$BASE_SHA" ] && [ -n "$HEAD_SHA" ]; then
              changed=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" || true)
              if printf '%s\n' "$changed" | grep -qE '^(\.github/workflows/.*\.yml|pyproject\.toml|uv\.lock|tools/scripts/(pgo_training|mypyc_smoke)\.py)$'; then
                mode=full
                reason="PR touches build-matrix infra"
              fi
            fi
          fi
          echo "matrix_mode=$mode" >> "$GITHUB_OUTPUT"
          echo "Resolved matrix_mode=$mode ($reason)"

  wait-for-cheap-ci:
    name: Wait for quality and unit checks
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: Skip wait for manual runs
        if: github.event_name != 'pull_request'
        run: echo "workflow_dispatch builds do not wait for PR checks"

      - name: Wait for Tests And Linting checks
        if: github.event_name == 'pull_request'
        env:
          GH_TOKEN: ${{ github.token }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
          REPOSITORY: ${{ github.repository }}
          REQUIRED_CHECKS: |
            quality
            test-unit (3.10)
            test-unit (3.11)
            test-unit (3.12)
            test-unit (3.13)
            test-unit (3.14)
        shell: bash
        run: |
          set -euo pipefail

          deadline=$((SECONDS + 25 * 60))

          while true; do
            response="$(gh api "repos/$REPOSITORY/commits/$HEAD_SHA/check-runs?per_page=100")"
            pending=0

            while IFS= read -r check_name; do
              [ -z "$check_name" ] && continue

              row="$(jq -r --arg name "$check_name" '
                [.check_runs[] | select(.name == $name)]
                | sort_by(.started_at // .created_at // "")
                | last
                | if . == null then "" else [.status, (.conclusion // ""), .html_url] | @tsv end
              ' <<< "$response")"

              if [ -z "$row" ]; then
                echo "Waiting for check to appear: $check_name"
                pending=$((pending + 1))
                continue
              fi

              IFS=$'\t' read -r status conclusion url <<< "$row"
              case "$status:$conclusion" in
                completed:success)
                  echo "Passed: $check_name"
                  ;;
                completed:*)
                  echo "::error::Required check '$check_name' completed with conclusion '$conclusion': $url"
                  exit 1
                  ;;
                *)
                  echo "Waiting for $check_name ($status): $url"
                  pending=$((pending + 1))
                  ;;
              esac
            done <<< "$REQUIRED_CHECKS"

            if [ "$pending" -eq 0 ]; then
              echo "Quality and unit checks passed for $HEAD_SHA"
              exit 0
            fi

            if [ "$SECONDS" -ge "$deadline" ]; then
              echo "::error::Timed out waiting for quality and unit checks for $HEAD_SHA"
              exit 1
            fi

            sleep 30
          done

  build-source:
    name: Build source distribution
    runs-on: ubuntu-latest
    needs: [wait-for-cheap-ci]
    steps:
      - name: Check out repository
        uses: actions/checkout@v7

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: uv.lock

      - name: Set up Python
        run: uv python install 3.12

      - name: Install dependencies
        run: uv sync --extra performance --group build

      - name: Build source distribution
        run: uv build --sdist

      - name: Upload source artifacts
        uses: actions/upload-artifact@v7
        with:
          name: source-dist
          path: dist/*.tar.gz

  build-wheels-standard:
    name: Build standard pure Python wheel
    runs-on: ubuntu-latest
    needs: [wait-for-cheap-ci]
    steps:
      - name: Check out repository
        uses: actions/checkout@v7

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: uv.lock

      - name: Set up Python 3.12
        run: uv python install 3.12

      - name: Install dependencies
        run: uv sync --extra performance --group build

      - name: Build standard wheel
        run: uv build --wheel

      - name: Upload wheel artifacts
        uses: actions/upload-artifact@v7
        with:
          name: wheels-standard
          path: dist/*.whl

  build-wheels-mypyc:
    name: Build MyPyC wheels (${{ matrix.os }} py${{ matrix.python-version }})
    runs-on: ${{ matrix.os }}
    needs: [detect-mode, wait-for-cheap-ci]
    strategy:
      fail-fast: true
      matrix:
        os: ${{ needs.detect-mode.outputs.matrix_mode == 'full' && fromJSON('["ubuntu-latest", "macos-latest", "windows-latest"]') || fromJSON('["ubuntu-latest"]') }}
        python-version:
          - ${{ github.event.inputs.python_version || '3.12' }}
    steps:
      - name: Check out repository
        uses: actions/checkout@v7

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.12"

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: uv.lock

      - name: Export build constraints from uv.lock
        run: uv export --frozen --no-emit-project --no-hashes --format requirements.txt --output-file build-constraints.txt

      - name: Cache pip downloads
        uses: actions/cache@v6
        with:
          path: |
            ~/.cache/pip
            ~/Library/Caches/pip
            ~\AppData\Local\pip\Cache
          key: pip-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('build-constraints.txt') }}
          restore-keys: |
            pip-${{ runner.os }}-py${{ matrix.python-version }}-

      - name: Cache mypyc build dir
        if: matrix.os == 'ubuntu-latest'
        uses: actions/cache@v6
        with:
          path: /tmp/sqlspec-mypyc-build
          key: mypyc-${{ matrix.os }}-py${{ matrix.python-version }}-${{ hashFiles('sqlspec/**/*.py', 'pyproject.toml') }}
          restore-keys: |
            mypyc-${{ matrix.os }}-py${{ matrix.python-version }}-

      - name: Pre-warm Windows Defender exclusions
        if: matrix.os == 'windows-latest'
        shell: pwsh
        run: |
          # Defender's real-time scan on the freshly-extracted nuget-cpython
          # python.exe blocks virtualenv discovery past its 10s timeout.
          # Keep mypyc's build root short enough for MSVC linker command lines.
          Remove-Item -Recurse -Force "C:\b" -ErrorAction SilentlyContinue
          New-Item -ItemType Directory -Force "C:\b" | Out-Null
          Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\pypa\cibuildwheel" -ErrorAction SilentlyContinue
          Add-MpPreference -ExclusionPath "$env:TEMP" -ErrorAction SilentlyContinue
          Add-MpPreference -ExclusionPath "C:\b" -ErrorAction SilentlyContinue
          Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE" -ErrorAction SilentlyContinue
          Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE\wheelhouse" -ErrorAction SilentlyContinue
          Add-MpPreference -ExclusionProcess "python.exe" -ErrorAction SilentlyContinue

      - name: Install cibuildwheel
        run: python -m pip install cibuildwheel

      - name: Build wheels with cibuildwheel
        run: python -m cibuildwheel --output-dir wheelhouse
        env:
          CIBW_BUILD: "cp${{ matrix.python-version == '3.10' && '310' || matrix.python-version == '3.11' && '311' || matrix.python-version == '3.12' && '312' || matrix.python-version == '3.13' && '313' || matrix.python-version == '3.14' && '314' }}-*"
          CIBW_BUILD_VERBOSITY: 1

          # PR validation builds host arch only — release covers full cross-arch matrix.
          CIBW_ARCHS_LINUX: "x86_64"
          CIBW_ARCHS_MACOS: "arm64"
          CIBW_ARCHS_WINDOWS: "AMD64"

          CIBW_SKIP: "cp39-win_arm64 *-musllinux*"

          CIBW_BEFORE_BUILD: "pip install -c {project}/build-constraints.txt hatch-mypyc hatchling"

          CIBW_TEST_REQUIRES: "cloud-sql-python-connector google-cloud-alloydb-connector"
          CIBW_TEST_COMMAND: >-
            python {project}/tools/scripts/mypyc_smoke.py --require-compiled

          # Linux PR validation exercises the same three-stage PGO path as publish.yml.
          CIBW_BEFORE_BUILD_LINUX: >-
            pip install -c {project}/build-constraints.txt hatch-mypyc hatchling &&
            PGO_DIR=/tmp/sqlspec-pgo &&
            BUILD_DIR=/tmp/sqlspec-mypyc-build &&
            mkdir -p "$PGO_DIR" "$BUILD_DIR" &&
            CFLAGS="-fprofile-generate=$PGO_DIR"
            HATCH_BUILD_HOOKS_ENABLE=1
            HATCH_MYPYC_BUILD_DIR="$BUILD_DIR"
            MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1
            pip install {package} --no-build-isolation &&
            pip install anyio pyarrow aiosqlite duckdb adbc-driver-sqlite adbc-driver-manager &&
            python {project}/tools/scripts/pgo_training.py &&
            pip uninstall -y sqlspec &&
            rm -rf "$BUILD_DIR/build" "$BUILD_DIR/tmp"

          CIBW_ENVIRONMENT_LINUX: >-
            HATCH_BUILD_HOOKS_ENABLE=1
            HATCH_MYPYC_BUILD_DIR=/tmp/sqlspec-mypyc-build
            MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1
            CFLAGS="-fprofile-use=/tmp/sqlspec-pgo -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch"

          # macOS + Windows: plain mypyc, no PGO. macOS PGO needs filter
          # deps absent from BEFORE_BUILD; MSVC PGO isn't validated.
          CIBW_ENVIRONMENT_MACOS: >-
            HATCH_BUILD_HOOKS_ENABLE=1
            MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1

          CIBW_ENVIRONMENT_WINDOWS: >-
            HATCH_BUILD_HOOKS_ENABLE=1
            HATCH_MYPYC_BUILD_DIR=C:/b
            MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1

      - name: Upload wheel artifacts
        uses: actions/upload-artifact@v7
        with:
          name: wheels-mypyc-${{ matrix.os }}-py${{ matrix.python-version }}
          path: wheelhouse/*.whl


  test-wheels:
    name: Test ${{ matrix.os }} py${{ matrix.python-version }}
    needs: [build-wheels-standard, build-wheels-mypyc, detect-mode]
    strategy:
      fail-fast: true
      matrix:
        os: ${{ needs.detect-mode.outputs.matrix_mode == 'full' && fromJSON('["ubuntu-latest", "windows-latest", "macos-latest"]') || fromJSON('["ubuntu-latest"]') }}
        python-version:
          - ${{ github.event.inputs.python_version || '3.12' }}

    runs-on: ${{ matrix.os }}
    steps:
      - name: Check out repository
        uses: actions/checkout@v7

      # PR builds have no release tag — derive expected version from
      # pyproject.toml so the version assertion below has something to check.
      - name: Resolve expected version
        shell: bash
        run: |
          v=$(grep -m1 '^version = ' pyproject.toml | sed -E 's/version = "(.*)"/\1/')
          echo "SQLSPEC_EXPECTED_VERSION=$v" >> "$GITHUB_ENV"

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: uv.lock

      - name: Set up Python ${{ matrix.python-version }}
        run: uv python install ${{ matrix.python-version }}

      - name: Download standard wheel artifacts
        uses: actions/download-artifact@v8
        with:
          name: wheels-standard
          path: dist-standard/

      - name: Download mypyc wheel artifacts
        uses: actions/download-artifact@v8
        with:
          pattern: wheels-mypyc-*
          merge-multiple: true
          path: dist-mypyc/

      - name: Test standard wheel installation
        shell: bash
        run: |
          uv venv test-standard --python ${{ matrix.python-version }}
          uv pip install --python test-standard --no-index --no-deps --find-links dist-standard/ "sqlspec==${SQLSPEC_EXPECTED_VERSION}"
          uv pip install --python test-standard "sqlspec==${SQLSPEC_EXPECTED_VERSION}" --find-links dist-standard/
          uv run --no-project --python test-standard python -I -c "import sqlspec, os, sys; v=sqlspec.__version__; e=os.environ['SQLSPEC_EXPECTED_VERSION']; sys.exit(0 if v == e else (sys.stderr.write(f'version mismatch: got {v}, expected {e}\n') or 1))"
          uv run --no-project --python test-standard python -I -c "import sqlspec; print('Standard wheel OK')"

      - name: Test mypyc wheel installation
        shell: bash
        run: |
          uv venv test-mypyc --python ${{ matrix.python-version }}
          uv pip install --python test-mypyc --no-index --no-deps --find-links dist-mypyc/ "sqlspec==${SQLSPEC_EXPECTED_VERSION}"
          uv pip install --python test-mypyc "sqlspec==${SQLSPEC_EXPECTED_VERSION}" --find-links dist-mypyc/
          uv run --no-project --python test-mypyc python -I -c "import sqlspec, os, sys; v=sqlspec.__version__; e=os.environ['SQLSPEC_EXPECTED_VERSION']; sys.exit(0 if v == e else (sys.stderr.write(f'version mismatch: got {v}, expected {e}\n') or 1))"
          uv run --no-project --python test-mypyc python -I tools/scripts/mypyc_smoke.py --require-compiled

  verify-packages:
    name: Verify package integrity
    needs: [build-source, build-wheels-standard, build-wheels-mypyc]
    runs-on: ubuntu-latest
    steps:
      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: uv.lock

      - name: Download all artifacts
        uses: actions/download-artifact@v8
        with:
          pattern: "*"
          merge-multiple: true
          path: dist/

      - name: List all built packages
        run: |
          echo "=== All built packages ==="
          find dist/ -name "*.whl" -o -name "*.tar.gz" | sort
          echo "=== Package count ==="
          find dist/ -name "*.whl" | wc -l
          find dist/ -name "*.tar.gz" | wc -l