Merge branch 'duckdb:main' into CastToUUID #4
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # This workflow is responsible for building all DuckDB extensions | |
| # | |
| name: Extensions (all platforms) | |
| on: | |
| workflow_call: | |
| inputs: | |
| override_git_describe: | |
| type: string | |
| git_ref: | |
| type: string | |
| skip_tests: | |
| type: string | |
| run_all: | |
| type: string | |
| workflow_dispatch: | |
| inputs: | |
| override_git_describe: | |
| description: 'Version tag to override git describe. Use to produce binaries' | |
| type: string | |
| git_ref: | |
| description: 'Set to override the DuckDB version, leave empty for current commit' | |
| type: string | |
| required: false | |
| default: '' | |
| extra_exclude_archs: | |
| description: 'Inject more architectures to skip' | |
| type: string | |
| required: false | |
| default: '' | |
| skip_tests: | |
| description: 'Set to true to skip all testing' | |
| type: boolean | |
| required: false | |
| default: false | |
| run_all: | |
| type: string | |
| required: false | |
| default: 'true' | |
| push: | |
| branches-ignore: | |
| - 'main' | |
| - 'feature' | |
| - 'v*.*-*' | |
| paths-ignore: | |
| - '**.md' | |
| - 'tools/**' | |
| - '!tools/shell/**' | |
| - '.github/patches/duckdb-wasm/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/Extensions.yml' | |
| - '!.github/workflows/_extension_distribution.yml' | |
| merge_group: | |
| pull_request: | |
| types: [opened, reopened, ready_for_review, converted_to_draft] | |
| paths-ignore: | |
| - '**.md' | |
| - 'tools/**' | |
| - '!tools/shell/**' | |
| - '.github/patches/duckdb-wasm/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/Extensions.yml' | |
| concurrency: | |
| group: extensions-${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}-${{ inputs.override_git_describe }} | |
| cancel-in-progress: true | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN }} | |
| jobs: | |
| check-draft: | |
| # We run all other jobs on PRs only if they are not draft PR | |
| if: github.event_name != 'pull_request' || github.event.pull_request.draft == false | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Preliminary checks on CI | |
| run: echo "Event name is ${{ github.event_name }}" | |
| # This first step loads the various extension configs from the ~/.github/config directory storing them to drive the build jobs | |
| load-extension-configs: | |
| name: Load Extension Configs | |
| runs-on: ubuntu-latest | |
| needs: check-draft | |
| outputs: | |
| main_extensions_config: ${{ steps.set-main-extensions.outputs.extension_config }} | |
| main_extensions_exclude_archs: ${{ steps.set-main-extensions.outputs.exclude_archs }} | |
| rust_based_extensions_config: ${{ steps.set-rust-based-extensions.outputs.extension_config }} | |
| rust_based_extensions_exclude_archs: ${{ steps.set-rust-based-extensions.outputs.exclude_archs }} | |
| env: | |
| # NOTE: on PRs we exclude some archs to speed things up | |
| BASE_EXCLUDE_ARCHS: ${{ (github.event_name == 'pull_request' || inputs.run_all != 'true') && 'wasm_eh;wasm_threads;windows_amd64_mingw;osx_amd64;linux_arm64;linux_amd64_musl;' || '' }} | |
| EXTRA_EXCLUDE_ARCHS: ${{ inputs.extra_exclude_archs }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - id: set-main-extensions | |
| name: Configure main extensions | |
| env: | |
| IN_TREE_CONFIG_FILE: .github/config/in_tree_extensions.cmake | |
| OUT_OF_TREE_CONFIG_FILE: .github/config/out_of_tree_extensions.cmake | |
| DEFAULT_EXCLUDE_ARCHS: '' | |
| run: | | |
| # Set config | |
| echo exclude_archs="$DEFAULT_EXCLUDE_ARCHS;$BASE_EXCLUDE_ARCHS;$EXTRA_EXCLUDE_ARCHS" >> $GITHUB_OUTPUT | |
| in_tree_extensions="`cat $IN_TREE_CONFIG_FILE`" | |
| out_of_tree_extensions="`cat $OUT_OF_TREE_CONFIG_FILE`" | |
| echo "extension_config<<EOF" >> $GITHUB_OUTPUT | |
| echo "$in_tree_extensions" >> $GITHUB_OUTPUT | |
| echo -e "\n" >> $GITHUB_OUTPUT | |
| echo "$out_of_tree_extensions" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| cat $GITHUB_OUTPUT | |
| - id: set-rust-based-extensions | |
| name: Configure Rust-based extensions | |
| env: | |
| CONFIG_FILE: .github/config/rust_based_extensions.cmake | |
| DEFAULT_EXCLUDE_ARCHS: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw;linux_amd64_musl' | |
| run: | | |
| echo exclude_archs="$DEFAULT_EXCLUDE_ARCHS;$BASE_EXCLUDE_ARCHS;$EXTRA_EXCLUDE_ARCHS" >> $GITHUB_OUTPUT | |
| rust_based_extensions="`cat .github/config/rust_based_extensions.cmake`" | |
| echo "extension_config<<EOF" >> $GITHUB_OUTPUT | |
| echo "$rust_based_extensions" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| cat $GITHUB_OUTPUT | |
| # Build the extensions from .github/config/in_tree_extensions.cmake | |
| main-extensions: | |
| name: Main Extensions | |
| needs: | |
| - load-extension-configs | |
| uses: ./.github/workflows/_extension_distribution.yml | |
| with: | |
| artifact_prefix: main-extensions | |
| exclude_archs: ${{ needs.load-extension-configs.outputs.main_extensions_exclude_archs }} | |
| extension_config: ${{ needs.load-extension-configs.outputs.main_extensions_config }} | |
| override_tag: ${{ inputs.override_git_describe }} | |
| duckdb_ref: ${{ inputs.git_ref }} | |
| skip_tests: ${{ inputs.skip_tests && true || false }} | |
| save_cache: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} | |
| # Build the extensions from .github/config/rust_based_extensions.cmake | |
| rust-based-extensions: | |
| name: Rust-based Extensions | |
| needs: | |
| - load-extension-configs | |
| uses: ./.github/workflows/_extension_distribution.yml | |
| with: | |
| exclude_archs: ${{ needs.load-extension-configs.outputs.rust_based_extensions_exclude_archs }} | |
| artifact_prefix: rust-based-extensions | |
| extension_config: ${{ needs.load-extension-configs.outputs.rust_based_extensions_config }} | |
| extra_toolchains: 'rust' | |
| override_tag: ${{ inputs.override_git_describe }} | |
| duckdb_ref: ${{ inputs.git_ref }} | |
| skip_tests: ${{ inputs.skip_tests && true || false }} | |
| save_cache: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} | |
| # Merge all extensions into a single, versioned repository | |
| create-extension-repository: | |
| name: Create Extension Repository | |
| runs-on: ubuntu-latest | |
| needs: | |
| - main-extensions | |
| - rust-based-extensions | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/download-artifact@v4 | |
| name: Download main extensions | |
| with: | |
| pattern: main-extensions-${{ github.sha }}* | |
| path: /tmp/repository_generation/main-extensions | |
| - uses: actions/download-artifact@v4 | |
| name: Download rust-based extensions | |
| with: | |
| pattern: rust-based-extensions-${{ github.sha }}* | |
| path: /tmp/repository_generation/rust-based-extensions | |
| - name: Print all extensions | |
| run: | | |
| tree /tmp/repository_generation | |
| - name: Merge into single repository | |
| run: | | |
| mkdir /tmp/merged_repository | |
| cp -r /tmp/repository_generation/*/*/* /tmp/merged_repository | |
| tree /tmp/merged_repository | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| if-no-files-found: error | |
| name: extension-repository-${{ github.sha }} | |
| path: | | |
| /tmp/merged_repository/**/*.duckdb_extension* | |
| upload-extensions: | |
| name: Upload Extensions | |
| runs-on: ubuntu-latest | |
| needs: | |
| - create-extension-repository | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: extension-repository-${{ github.sha }} | |
| path: /tmp | |
| - name: List extensions to deploy | |
| shell: bash | |
| run: | | |
| tree /tmp/extension-repository-${{ github.sha }} | |
| - name: Deploy extensions | |
| shell: bash | |
| env: | |
| AWS_ENDPOINT_URL: ${{ secrets.DUCKDB_CORE_EXTENSION_S3_ENDPOINT }} | |
| AWS_ACCESS_KEY_ID: ${{secrets.DUCKDB_CORE_EXTENSION_S3_ID}} | |
| AWS_SECRET_ACCESS_KEY: ${{secrets.DUCKDB_CORE_EXTENSION_S3_SECRET}} | |
| DUCKDB_DEPLOY_SCRIPT_MODE: for_real | |
| DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }} | |
| run: | | |
| pip install awscli | |
| ./scripts/extension-upload-repository.sh /tmp/extension-repository-${{ github.sha }} | |
| autoload-tests: | |
| name: Extension Autoloading Tests | |
| if: ${{ !inputs.skip_tests}} | |
| runs-on: ubuntu-latest | |
| needs: create-extension-repository | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - name: Setup Build Environment | |
| run: | | |
| sudo apt-get update -y -qq | |
| sudo apt-get install -y -qq ninja-build ccache | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: extension-repository-${{ github.sha }} | |
| path: /tmp | |
| - name: List extensions to test with | |
| shell: bash | |
| run: | | |
| tree /tmp/extension-repository-${{ github.sha }} | |
| - name: Build DuckDB | |
| env: | |
| GEN: ninja | |
| CC: gcc | |
| CXX: g++ | |
| EXTENSION_CONFIGS: './.github/config/rust_based_extensions.cmake;./.github/config/out_of_tree_extensions.cmake;./.github/config/in_tree_extensions.cmake' | |
| EXTENSION_TESTS_ONLY: 1 | |
| ENABLE_EXTENSION_AUTOLOADING: 1 | |
| ENABLE_EXTENSION_AUTOINSTALL: 1 | |
| run: | | |
| make release | |
| - name: Run Tests | |
| env: | |
| LOCAL_EXTENSION_REPO: /tmp/extension-repository-${{ github.sha }} | |
| run: | | |
| ./build/release/test/unittest --autoloading available --skip-compiled | |
| check-load-install-extensions: | |
| name: Checks extension entries | |
| if: ${{ !inputs.skip_tests}} | |
| runs-on: ubuntu-22.04 | |
| needs: create-extension-repository | |
| env: | |
| CC: gcc-10 | |
| CXX: g++-10 | |
| GEN: ninja | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - name: Install | |
| shell: bash | |
| run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} | |
| - name: Build | |
| shell: bash | |
| env: | |
| GENERATE_EXTENSION_ENTRIES: 1 | |
| LOCAL_EXTENSION_REPO: build/release/repository_other | |
| run: | | |
| make | |
| - uses: actions/download-artifact@v4 | |
| name: Download extension repository artifact | |
| with: | |
| pattern: extension-repository-${{ github.sha }} | |
| path: /tmp | |
| - name: Copy over local extension repository | |
| shell: bash | |
| run: | | |
| cp -r /tmp/extension-repository-${{ github.sha }} build/release/repository | |
| tree build/release/repository | |
| find build/release/repository -type f ! -path "build/release/repository/*/linux_amd64/*" -delete | |
| tree build/release/repository | |
| - name: Check if extension_entries.hpp is up to date | |
| shell: bash | |
| env: | |
| EXTENSION_CONFIGS: '.github/config/in_tree_extensions.cmake;.github/config/out_of_tree_extensions.cmake' | |
| run: | | |
| make extension_configuration | |
| python scripts/generate_extensions_function.py | |
| pip install "black>=24" | |
| pip install cmake-format | |
| pip install "clang_format==11.0.1" | |
| make format-fix | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: extension_entries.hpp | |
| path: | | |
| src/include/duckdb/main/extension_entries.hpp | |
| - name: Check for any difference | |
| run: | | |
| git diff --exit-code src/include/duckdb/main/extension_entries.hpp && echo "No differences found" | |
| - name: Explainer | |
| if: failure() | |
| run: | | |
| echo "There are differences in src/include/duckdb/main/extension_entries.hpp" | |
| echo "Check the uploaded extension_entries.hpp (in the workflow Summary), and check that in instead of src/include/duckdb/main/extension_entries.hpp" |