LinuxRelease.yml

name: LinuxRelease
on:
  workflow_call:
    inputs:
      override_git_describe:
        type: string
      git_ref:
        type: string
      skip_tests:
        type: string
  workflow_dispatch:
    inputs:
      override_git_describe:
        description: 'Version tag to override git describe'
        type: string
      git_ref:
        description: 'Git ref'
        type: string
      skip_tests:
        description: 'Pass "true" to skip tests'
        type: string


concurrency:
  group: linuxrelease-${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}-${{ inputs.override_git_describe }}
  cancel-in-progress: true

env:
  GH_TOKEN: ${{ secrets.GH_TOKEN }}
  OVERRIDE_GIT_DESCRIBE: ${{ inputs.override_git_describe }}

jobs:
 check-draft:
    # We run all other jobs on PRs only if they are not draft PR
    if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
    runs-on: ubuntu-24.04
    steps:
      - name: Preliminary checks on CI
        run: echo "Event name is ${{ github.event_name }}"

 linux-release-cli:
    needs:
      - check-draft

    strategy:
      fail-fast: false
      matrix:
        config: [ { runner: ubuntu-latest, arch: amd64, image: x86_64}, {runner: ubuntu-24.04-arm, arch: arm64, image: aarch64}]

    name: Linux CLI (${{ matrix.config.arch }})
    runs-on: ${{ matrix.config.runner }}

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        ref: ${{ inputs.git_ref }}

    - uses: ./.github/actions/cleanup_runner

    - name: Install pytest
      run: |
        python3 -m pip install pytest

    - name: Build
      shell: bash
      run: |
        export PWD=`pwd`
        docker run                                                             \
        -v$PWD:$PWD                                                            \
        -e GEN=ninja                                                           \
        -e CC='ccache gcc'                                                     \
        -e CXX='ccache g++'                                                    \
        -e CCACHE_DIR=$PWD/.ccache                                             \
        -e OVERRIDE_GIT_DESCRIBE=$OVERRIDE_GIT_DESCRIBE                        \
        -e EXTENSION_CONFIGS="$PWD/.github/config/bundled_extensions.cmake"    \
        -e ENABLE_EXTENSION_AUTOLOADING=1                                      \
        -e ENABLE_EXTENSION_AUTOINSTALL=1                                      \
        -e BUILD_BENCHMARK=1                                                   \
        -e FORCE_WARN_UNUSED=1                                                 \
        quay.io/pypa/manylinux_2_28_${{ matrix.config.image }}                 \
        bash -c "
          set -e
          cat /etc/os-release
          dnf install -y \
            ccache       \
            ninja-build  \
            perl-IPC-Cmd
          git config --global --add safe.directory $PWD
          make -C $PWD
        "

    - name: Print platform
      shell: bash
      run: ./build/release/duckdb -c "PRAGMA platform;"

    - name: Deploy
      shell: bash
      env:
        AWS_ENDPOINT_URL: ${{ secrets.S3_DUCKDB_STAGING_ENDPOINT }}
        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
      run: |
        python3 scripts/amalgamation.py
        zip -j duckdb_cli-linux-${{ matrix.config.arch }}.zip build/release/duckdb
        gzip -9 -k -n -c build/release/duckdb > duckdb_cli-linux-${{ matrix.config.arch }}.gz
        zip -j libduckdb-linux-${{ matrix.config.arch }}.zip build/release/src/libduckdb*.* src/amalgamation/duckdb.hpp src/include/duckdb.h
        ./scripts/upload-assets-to-staging.sh github_release libduckdb-linux-${{ matrix.config.arch }}.zip duckdb_cli-linux-${{ matrix.config.arch }}.zip duckdb_cli-linux-${{ matrix.config.arch }}.gz

    - uses: actions/upload-artifact@v4
      with:
        name: duckdb-binaries-linux-${{ matrix.config.arch }}
        path: |
          libduckdb-linux-${{ matrix.config.arch }}.zip
          duckdb_cli-linux-${{ matrix.config.arch }}.zip
          duckdb_cli-linux-${{ matrix.config.arch }}.gz

    - name: Test
      shell: bash
      if: ${{ inputs.skip_tests != 'true' }}
      run: |
        python3 scripts/run_tests_one_by_one.py build/release/test/unittest "*" --time_execution

    - name: Release specific tests
      shell: bash
      run: |
        ./build/release/test/unittest --select-tag release

    - name: Tools Tests
      shell: bash
      run: |
        python3 -m pytest tools/shell/tests --shell-binary build/release/duckdb

    - name: Examples
      shell: bash
      run: |
        build/release/benchmark/benchmark_runner benchmark/micro/update/update_with_join.benchmark
        build/release/duckdb -c "COPY (SELECT 42) TO '/dev/stdout' (FORMAT PARQUET)" | cat

 linux-musl-release-cli:
    needs:
      - check-draft

    strategy:
      fail-fast: false
      matrix:
        config: [ { runner: ubuntu-latest, arch: amd64}, {runner: ubuntu-24.04-arm, arch: arm64}]

    name: Linux CLI (${{ matrix.config.arch }}-musl)
    runs-on: ${{ matrix.config.runner }}

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        ref: ${{ inputs.git_ref }}

    - uses: ./.github/actions/cleanup_runner

    - name: Build
      shell: bash
      run: |
        export PWD=`pwd`
        docker run                                                          \
        -v$PWD:$PWD                                                         \
        -e GEN=ninja                                                        \
        -e CC='ccache gcc'                                                  \
        -e CXX='ccache g++'                                                 \
        -e CCACHE_DIR=$PWD/.ccache                                          \
        -e OVERRIDE_GIT_DESCRIBE=$OVERRIDE_GIT_DESCRIBE                     \
        -e EXTENSION_CONFIGS="$PWD/.github/config/bundled_extensions.cmake" \
        -e ENABLE_EXTENSION_AUTOLOADING=1                                   \
        -e ENABLE_EXTENSION_AUTOINSTALL=1                                   \
        -e BUILD_BENCHMARK=1                                                \
        -e FORCE_WARN_UNUSED=1                                              \
        alpine:3.22                                                         \
        sh -c "
          set -e
          cat /etc/os-release
          apk add   \
            ccache  \
            cmake   \
            g++     \
            git     \
            make    \
            samurai
          git config --global --add safe.directory $PWD
          make -C $PWD
        "

    - name: Deploy
      shell: bash
      env:
        AWS_ENDPOINT_URL: ${{ secrets.S3_DUCKDB_STAGING_ENDPOINT }}
        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
      run: |
        python3 scripts/amalgamation.py
        zip -j duckdb_cli-linux-${{ matrix.config.arch }}-musl.zip build/release/duckdb
        gzip -9 -k -n -c build/release/duckdb > duckdb_cli-linux-${{ matrix.config.arch }}-musl.gz
        zip -j libduckdb-linux-${{ matrix.config.arch }}-musl.zip build/release/src/libduckdb*.* src/amalgamation/duckdb.hpp src/include/duckdb.h
        ./scripts/upload-assets-to-staging.sh github_release libduckdb-linux-${{ matrix.config.arch }}-musl.zip duckdb_cli-linux-${{ matrix.config.arch }}-musl.zip duckdb_cli-linux-${{ matrix.config.arch }}-musl.gz

    - uses: actions/upload-artifact@v4
      with:
        name: duckdb-binaries-linux-${{ matrix.config.arch }}-musl
        path: |
          libduckdb-linux-${{ matrix.config.arch }}-musl.zip
          duckdb_cli-linux-${{ matrix.config.arch }}-musl.zip
          duckdb_cli-linux-${{ matrix.config.arch }}-musl.gz

    - name: Test
      shell: bash
      if: ${{ inputs.skip_tests != 'true' }}
      run: |
        export PWD=`pwd`
        docker run   \
        -v$PWD:$PWD  \
        alpine:3.22                                                         \
        sh -c "
          set -e
          cat /etc/os-release
          apk add      \
            python3    \
            py3-pytest
          cd $PWD
          echo Tests
          python3 scripts/run_tests_one_by_one.py build/release/test/unittest \"*\" --time_execution
          echo Release specific tests
          ./build/release/test/unittest --select-tag release
          echo Tools Tests
          python3 -m pytest tools/shell/tests --shell-binary build/release/duckdb
          echo Examples
          build/release/benchmark/benchmark_runner benchmark/micro/update/update_with_join.benchmark
          build/release/duckdb -c \"COPY (SELECT 42) TO '/dev/stdout' (FORMAT PARQUET)\" | cat
        "

 main_julia:
    name: Julia ${{ matrix.version }}
    needs: linux-release-cli
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        version:
          - '1.10'
          - '1'
        arch:
          - x64

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        ref: ${{ inputs.git_ref }}

    - uses: julia-actions/setup-julia@v2
      with:
        version: ${{ matrix.version }}
        arch: ${{ matrix.arch }}

    - uses: actions/download-artifact@v4
      with:
        name: duckdb-binaries-linux-amd64
        path: linux-release-artifacts

    - name: Extract DuckDB Linux release artifacts
      shell: bash
      run: |
        mkdir -p build/release/src
        unzip -j linux-release-artifacts/duckdb_cli-linux-amd64.zip "duckdb" -d build/release
        chmod +x build/release/duckdb
        unzip -j linux-release-artifacts/libduckdb-linux-amd64.zip "libduckdb.so*" -d build/release/src
        ls -lah build/release/duckdb
        ls -lah build/release/src/libduckdb.so*

    - name: Run Tests
      shell: bash
      run: |
        export JULIA_DUCKDB_LIBRARY="`pwd`/build/release/src/libduckdb.so"
        export JULIA_NUM_THREADS=2
        export LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libstdc++.so.6"
        ls $JULIA_DUCKDB_LIBRARY
        cd tools/juliapkg
        julia --project -e "import Pkg; Pkg.test()"

    - name: Format Check
      if: ${{ always() && matrix.version == '1.10' }}
      shell: bash
      run: |
        cd tools/juliapkg
        julia -e "import Pkg; Pkg.add(\"JuliaFormatter\")"
        ./format_check.sh

 upload-libduckdb-src:
    name: Upload libduckdb-src.zip
    needs: linux-release-cli
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        ref: ${{ inputs.git_ref }}

    - name: Deploy
      shell: bash
      env:
        AWS_ENDPOINT_URL: ${{ secrets.S3_DUCKDB_STAGING_ENDPOINT }}
        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
      run: |
        python3 scripts/amalgamation.py
        zip -j libduckdb-src.zip src/amalgamation/duckdb.hpp src/amalgamation/duckdb.cpp src/include/duckdb.h src/include/duckdb_extension.h
        ./scripts/upload-assets-to-staging.sh github_release libduckdb-src.zip

 symbol-leakage:
    name: Symbol Leakage
    runs-on: ubuntu-22.04
    needs: linux-release-cli
    if: ${{ inputs.skip_tests != 'true' }}
    env:
      GEN: ninja

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        ref: ${{ inputs.git_ref }}

    - uses: actions/setup-python@v5
      with:
        python-version: '3.12'

    - name: Install Ninja
      shell: bash
      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build

    - uses: ./.github/actions/ccache-action

    - name: Build
      shell: bash
      run: make

    - name: Symbol Leakage Test
      shell: bash
      run: python3 scripts/exported_symbols_check.py build/release/src/libduckdb*.so

    - name: Banned Symbol Check
      shell: bash
      run: python3 scripts/banned_symbols_check.py --directory build/release/src