Fix Path Typo in Extension's CMake Warning #2
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python | |
| on: | |
| workflow_call: | |
| inputs: | |
| override_git_describe: | |
| type: string | |
| git_ref: | |
| type: string | |
| skip_tests: | |
| type: string | |
| run_all: | |
| type: string | |
| override_twine_upload: | |
| type: string | |
| workflow_dispatch: | |
| inputs: | |
| override_git_describe: | |
| description: 'Version tag to override git describe' | |
| type: string | |
| git_ref: | |
| description: 'Git ref' | |
| type: string | |
| skip_tests: | |
| description: 'Pass "true" to skip tests' | |
| type: string | |
| run_all: | |
| description: 'Pass "true" to run all jobs' | |
| type: string | |
| override_twine_upload: | |
| description: 'Pass "true" to trigger TwineUpload.yml when not on tag or in "main"' | |
| type: string | |
| push: | |
| branches-ignore: | |
| - 'main' | |
| - 'feature' | |
| - 'v*.*-*' | |
| paths-ignore: | |
| - '**.md' | |
| - 'examples/**' | |
| - 'test/**' | |
| - 'tools/**' | |
| - '!tools/pythonpkg/**' | |
| - '.github/patches/duckdb-wasm/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/Python.yml' | |
| - '.github/config/out_of_tree_extensions.cmake' | |
| merge_group: | |
| pull_request: | |
| types: [opened, reopened, ready_for_review, converted_to_draft] | |
| paths-ignore: | |
| - '**.md' | |
| - 'examples/**' | |
| - 'test/**' | |
| - 'tools/**' | |
| - '!tools/pythonpkg/**' | |
| - '.github/patches/duckdb-wasm/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/Python.yml' | |
| - '.github/config/out_of_tree_extensions.cmake' | |
| concurrency: | |
| group: python-${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}-${{ inputs.override_git_describe }} | |
| cancel-in-progress: true | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN }} | |
| OVERRIDE_GIT_DESCRIBE: ${{ inputs.override_git_describe }} | |
| CIBW_TEST_SKIP: ${{ inputs.skip_tests == 'true' && '*-*' || '{cp37,cp38}-*' }} | |
| jobs: | |
| check-draft: | |
| # We run all other jobs on PRs only if they are not draft PR | |
| if: github.event_name != 'pull_request' || github.event.pull_request.draft == false | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Preliminary checks on CI | |
| run: echo "Event name is ${{ github.event_name }}" | |
| # This is just a sanity check of Python 3.10 running with Arrow / Spark | |
| linux-python3-10: | |
| name: Python 3.10 Linux | |
| needs: check-draft | |
| runs-on: ubuntu-22.04 | |
| env: | |
| CIBW_BUILD: cp310-manylinux_x86_64 | |
| CIBW_ARCHS: 'x86_64' | |
| CIBW_TEST_COMMAND: 'python -m pytest {project}/tests --verbose' | |
| # FIXME: add this back to CIBW_TEST_COMMAND '&& USE_ACTUAL_SPARK=true JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java)))) SPARK_HOME={project}/spark_installation/spark python -m pytest {project}/tests/fast/spark --verbose' | |
| PYTEST_TIMEOUT: '600' | |
| CIBW_ENVIRONMENT: 'OVERRIDE_GIT_DESCRIBE=${{ inputs.override_git_describe }}' | |
| CIBW_BEFORE_ALL: 'cd {project} && ./scripts/install_spark_in_cibuildwheels_linux_container.sh' | |
| CIBW_MANYLINUX_X86_64_IMAGE: 'manylinux_2_28' | |
| CIBW_MANYLINUX_PYPY_X86_64_IMAGE: 'manylinux_2_28' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - name: Check/Act on inputs.override_git_describe | |
| shell: bash | |
| run: | | |
| if [[ "${{ inputs.override_git_describe }}" == *-* ]]; then | |
| echo "override_git_describe ${{ inputs.override_git_describe }}: provide either vX.Y.Z or empty string" | |
| exit 1 | |
| elif [[ -z "${{ inputs.override_git_describe }}" ]]; then | |
| echo "No override_git_describe provided" | |
| else | |
| echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git log -1 --format=%h)" >> "$GITHUB_ENV" | |
| echo "override_git_describe ${{ inputs.override_git_describe }}: add tag" | |
| git tag ${{ inputs.override_git_describe }} | |
| fi | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install | |
| shell: bash | |
| run: pip install 'cibuildwheel>=2.16.2' build | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build source dist | |
| shell: bash | |
| working-directory: tools/pythonpkg | |
| run: | | |
| pyproject-build . --sdist | |
| mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball | |
| - name: Build | |
| shell: bash | |
| working-directory: tools/pythonpkg | |
| run: | | |
| export DISTUTILS_C_COMPILER_LAUNCHER=ccache | |
| # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030 | |
| cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball | |
| linux-python3: | |
| name: Python 3 Linux | |
| runs-on: ${{ matrix.platform.os }} | |
| needs: linux-python3-10 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: | |
| - { os: ubuntu-24.04, arch: x86_64 } | |
| - { os: ubuntu-24.04-arm, arch: aarch64 } | |
| python: [ cp39, cp310, cp311, cp312, cp313] | |
| isRelease: | |
| - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || inputs.run_all == 'true' }} | |
| exclude: | |
| # Speed things up a bit for non-releases | |
| - { isRelease: false, python: cp310 } | |
| - { isRelease: false, python: cp311 } | |
| - { isRelease: false, python: cp312 } | |
| env: | |
| CIBW_BUILD: ${{ matrix.python }}-manylinux_${{ matrix.platform.arch }} | |
| CIBW_ARCHS: ${{ matrix.platform.arch }} | |
| CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 | |
| CIBW_MANYLINUX_PYPY_AARCH64_IMAGE: manylinux_2_28 | |
| CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 | |
| CIBW_MANYLINUX_PYPY_X86_64_IMAGE: manylinux_2_28 | |
| PYTEST_TIMEOUT: '600' | |
| DUCKDB_BUILD_UNITY: 1 | |
| CIBW_ENVIRONMENT: 'OVERRIDE_GIT_DESCRIBE=${{ inputs.override_git_describe }}' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - name: Check/Act on inputs.override_git_describe | |
| shell: bash | |
| run: | | |
| if [[ "${{ inputs.override_git_describe }}" == *-* ]]; then | |
| echo "override_git_describe ${{ inputs.override_git_describe }}: provide either vX.Y.Z or empty string" | |
| exit 1 | |
| elif [[ -z "${{ inputs.override_git_describe }}" ]]; then | |
| echo "No override_git_describe provided" | |
| else | |
| echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git log -1 --format=%h)" >> "$GITHUB_ENV" | |
| echo "override_git_describe ${{ inputs.override_git_describe }}: add tag" | |
| git tag ${{ inputs.override_git_describe }} | |
| fi | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install | |
| shell: bash | |
| run: | | |
| pip install 'cibuildwheel>=2.16.2' build | |
| python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true" | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/[email protected] # Note: pinned due to GLIBC incompatibility in later releases | |
| with: | |
| key: ${{ github.job }}-${{ matrix.platform.arch }}-${{ matrix.python }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: | | |
| cd tools/pythonpkg | |
| pyproject-build . --sdist | |
| mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball | |
| ls duckdb_tarball | |
| export DISTUTILS_C_COMPILER_LAUNCHER=ccache | |
| # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030 | |
| if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then | |
| export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py --verbose' | |
| fi | |
| cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball | |
| ls wheelhouse | |
| - name: Deploy | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} | |
| shell: bash | |
| run: | | |
| cp tools/pythonpkg/dist/duckdb-*.tar.gz duckdb_python_src.tar.gz | |
| ./scripts/upload-assets-to-staging.sh github_release duckdb_python_src.tar.gz | |
| ./scripts/upload-assets-to-staging.sh twine_upload tools/pythonpkg/dist/duckdb-*.tar.gz tools/pythonpkg/wheelhouse/*.whl | |
| osx-python3: | |
| name: Python 3 OSX | |
| needs: linux-python3-10 | |
| if: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || inputs.run_all == 'true' }} | |
| runs-on: macos-latest | |
| strategy: | |
| matrix: | |
| python_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*] | |
| env: | |
| CIBW_BUILD: ${{ matrix.python_build}} | |
| CIBW_ARCHS: 'x86_64 universal2 arm64' | |
| CIBW_TEST_COMMAND: 'python -m pytest {project}/tests/fast --verbose' | |
| CIBW_ENVIRONMENT: 'MACOSX_DEPLOYMENT_TARGET=12.0' | |
| DUCKDB_BUILD_UNITY: 1 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - name: Check/Act on inputs.override_git_describe | |
| shell: bash | |
| run: | | |
| if [[ "${{ inputs.override_git_describe }}" == *-* ]]; then | |
| echo "override_git_describe ${{ inputs.override_git_describe }}: provide either vX.Y.Z or empty string" | |
| exit 1 | |
| elif [[ -z "${{ inputs.override_git_describe }}" ]]; then | |
| echo "No override_git_describe provided" | |
| else | |
| echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git log -1 --format=%h)" >> "$GITHUB_ENV" | |
| echo "override_git_describe ${{ inputs.override_git_describe }}: add tag" | |
| git tag ${{ inputs.override_git_describe }} | |
| fi | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install | |
| shell: bash | |
| run: pip install 'cibuildwheel>=2.16.2' build | |
| - name: Build | |
| shell: bash | |
| run: | | |
| cd tools/pythonpkg | |
| pyproject-build . --sdist | |
| mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball | |
| # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030 | |
| if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then | |
| export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py --verbose' | |
| fi | |
| cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball | |
| - name: Deploy | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} | |
| shell: bash | |
| run: | | |
| ./scripts/upload-assets-to-staging.sh twine_upload tools/pythonpkg/wheelhouse/*.whl | |
| win-python3: | |
| name: Python 3 Windows | |
| runs-on: windows-latest | |
| needs: linux-python3-10 | |
| strategy: | |
| matrix: | |
| python_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*] | |
| isRelease: | |
| - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || inputs.run_all == 'true' }} | |
| exclude: | |
| - isRelease: false | |
| python_build: 'cp38-*' | |
| - isRelease: false | |
| python_build: 'cp39-*' | |
| - isRelease: false | |
| python_build: 'cp310-*' | |
| - isRelease: false | |
| python_build: 'cp311-*' | |
| - isRelease: false | |
| python_build: 'cp312-*' | |
| env: | |
| CIBW_BUILD: ${{ matrix.python_build}} | |
| DUCKDB_BUILD_UNITY: 1 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - name: Check/Act on inputs.override_git_describe | |
| shell: bash | |
| run: | | |
| if [[ "${{ inputs.override_git_describe }}" == *-* ]]; then | |
| echo "override_git_describe ${{ inputs.override_git_describe }}: provide either vX.Y.Z or empty string" | |
| exit 1 | |
| elif [[ -z "${{ inputs.override_git_describe }}" ]]; then | |
| echo "No override_git_describe provided" | |
| else | |
| echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git log -1 --format=%h)" >> "$GITHUB_ENV" | |
| echo "override_git_describe ${{ inputs.override_git_describe }}: add tag" | |
| git tag ${{ inputs.override_git_describe }} | |
| fi | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install | |
| shell: bash | |
| run: pip install 'cibuildwheel>=2.16.2' build | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }}-${{ matrix.python_build }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build source dist | |
| shell: bash | |
| working-directory: tools/pythonpkg | |
| run: | | |
| pyproject-build . --sdist | |
| mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball | |
| - name: Build wheels | |
| shell: bash | |
| working-directory: tools/pythonpkg | |
| run: | | |
| export DISTUTILS_C_COMPILER_LAUNCHER=ccache | |
| # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030 | |
| if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then | |
| export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py --verbose' | |
| fi | |
| cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball | |
| - name: Deploy | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} | |
| shell: bash | |
| run: | | |
| ./scripts/upload-assets-to-staging.sh twine_upload tools/pythonpkg/wheelhouse/*.whl | |
| linux-release-cleanup: | |
| if: github.ref == 'refs/heads/v1.3-ossivalis' || github.ref == 'refs/heads/main' | |
| name: PyPi Release Cleanup | |
| needs: | |
| - twine-upload | |
| runs-on: ubuntu-22.04 | |
| env: | |
| PYPI_CLEANUP_USERNAME: 'mytherin' | |
| PYPI_CLEANUP_PASSWORD: ${{secrets.PYPI_CLEANUP_PASSWORD}} | |
| PYPI_CLEANUP_OTP: ${{secrets.PYPI_CLEANUP_OTP}} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install Packages | |
| shell: bash | |
| run: python3 -m pip install pyotp requests urllib3 | |
| - name: Cleanup Releases | |
| shell: bash | |
| if: ${{ github.repository == 'duckdb/duckdb' }} | |
| run: python3 scripts/pypi_cleanup.py | |
| twine-upload: | |
| needs: | |
| - osx-python3 | |
| - win-python3 | |
| - linux-python3 | |
| # Note that want to run this by default ONLY if no override_git_describe is provided | |
| # This means we are not staging a release, or we explicitly set `twine_upload` | |
| # Why? If present, it means we are staging releases, and we want to do the upload manually | |
| if: | | |
| ${{ ((( startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || inputs.override_twine_upload == 'true' )) && | |
| (( inputs.override_git_describe == '' )) && (( github.repository == 'duckdb/duckdb' ))) }} | |
| uses: ./.github/workflows/TwineUpload.yml | |
| secrets: inherit | |
| with: | |
| twine_upload: 'true' |