Main.yml

name: Main
on:
  workflow_dispatch:
  repository_dispatch:
  push:
    branches-ignore:
      - 'main'
      - 'feature'
      - 'v*.*-*'
    paths-ignore:
      - '**.md'
      - 'tools/**'
      - '!tools/shell/**'
      - '.github/patches/duckdb-wasm/**'
      - '.github/workflows/**'
      - '!.github/workflows/Main.yml'
      - '.github/config/extensions/*.cmake'
      - '.github/patches/extensions/**/*.patch'
      - '!.github/patches/extensions/fts/*.patch' # fts used in some jobs
      - '!.github/config/extensions/fts.cmake'
  merge_group:
  pull_request:
    types: [opened, reopened, ready_for_review, converted_to_draft]
    paths-ignore:
      - '**.md'
      - 'tools/**'
      - '!tools/shell/**'
      - '.github/patches/duckdb-wasm/**'
      - '.github/workflows/**'
      - '!.github/workflows/Main.yml'
      - '.github/config/extensions/*.cmake'
      - '.github/patches/extensions/**/*.patch'
      - '!.github/patches/extensions/fts/*.patch' # fts used in some jobs
      - '!.github/config/extensions/fts.cmake'


concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
  cancel-in-progress: true

env:
  GH_TOKEN: ${{ secrets.GH_TOKEN }}

jobs:
 check-draft:
    # We run all other jobs on PRs only if they are not draft PR
    if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
    runs-on: ubuntu-24.04
    steps:
      - name: Preliminary checks on CI
        run: echo "Event name is ${{ github.event_name }}"

 linux-debug:
    name: Linux Debug
    # This tests release build while enabling slow verifiers (masked by #ifdef DEBUG) and sanitizers
    needs: check-draft
    runs-on: ubuntu-22.04
    env:
      CC: gcc-10
      CXX: g++-10
      TREAT_WARNINGS_AS_ERRORS: 1
      GEN: ninja
      CRASH_ON_ASSERT: 1
      FORCE_DEBUG: 1
      FORCE_ASSERT: 1
      REDUCE_SYMBOLS: 1

    steps:
    - uses: actions/checkout@v4

    - id: describe_step
      run: echo "git_describe=$(git describe --tags --long)" >> "$GITHUB_OUTPUT"

    - name: Install
      shell: bash
      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build

    - uses: ./.github/actions/ccache-action

    - name: Build
      shell: bash
      run:  make relassert

    - name: Output version info
      shell: bash
      run: ./build/relassert/duckdb -c "PRAGMA version;"

    - name: Set DUCKDB_INSTALL_LIB for ADBC tests
      shell: bash
      run: echo "DUCKDB_INSTALL_LIB=$(find `pwd` -name "libduck*.so" | head -n 1)" >> $GITHUB_ENV

    - name: Test DUCKDB_INSTALL_LIB variable
      run: echo $DUCKDB_INSTALL_LIB

    - name: Test
      shell: bash
      run: |
        python3 scripts/run_tests_one_by_one.py build/relassert/test/unittest --tests-per-invocation 100 --timeout 300

 linux-release:
    name: Linux Release (full suite)
    needs: linux-debug
    runs-on: ubuntu-24.04
    env:
      GEN: ninja
      BUILD_JEMALLOC: 1
      CORE_EXTENSIONS: "icu;tpch;tpcds;fts;json;inet"
      DISABLE_SANITIZER: 1

    steps:
    - uses: actions/checkout@v4

    - name: Install
      shell: bash
      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build

    - uses: ./.github/actions/ccache-action

    - name: Build
      shell: bash
      run: make release

    - name: Test
      shell: bash
      run: make allunit

 no-string-inline:
    name: No String Inline / Destroy Unpinned Blocks
    runs-on: ubuntu-24.04
    needs: linux-debug
    env:
      GEN: ninja
      CORE_EXTENSIONS: "icu;parquet;tpch;tpcds;fts;json;inet"
      DISABLE_STRING_INLINE: 1
      DESTROY_UNPINNED_BLOCKS: 1
      ALTERNATIVE_VERIFY: 1
      DISABLE_POINTER_SALT: 1
      LSAN_OPTIONS: suppressions=${{ github.workspace }}/.sanitizer-leak-suppressions.txt
      DUCKDB_TEST_DESCRIPTION: 'Compiled with ALTERNATIVE_VERIFY=1 DISABLE_STRING_INLINE=1 DESTROY_UNPINNED_BLOCKS=1 DISABLE_POINTER_SALT=1. Use require no_alternative_verify to skip.'

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - uses: ./.github/actions/cleanup_runner

      - name: Install
        shell: bash
        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build

      - uses: ./.github/actions/ccache-action

      - name: Build
        shell: bash
        run: make relassert

      - name: Test
        shell: bash
        run: build/relassert/test/unittest

 vector-sizes:
    name: Vector Sizes
    runs-on: ubuntu-22.04
    needs: linux-debug
    env:
      CC: gcc-10
      CXX: g++-10
      GEN: ninja
      DUCKDB_TEST_DESCRIPTION: 'Compiled with STANDARD_VECTOR_SIZE=2. Use require vector_size 2048 to skip tests.'

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - uses: ./.github/actions/cleanup_runner

      - name: Install
        shell: bash
        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build

      - uses: ./.github/actions/ccache-action

      - name: Build
        shell: bash
        run: STANDARD_VECTOR_SIZE=2 make reldebug

      - name: Test
        shell: bash
        run: |
          python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest --no-exit --time_execution

 valgrind:
    name: Valgrind
    if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
    runs-on: ubuntu-24.04
    needs: linux-debug
    env:
      CC: clang
      CXX: clang++
      DISABLE_SANITIZER: 1
      BUILD_JEMALLOC: 1
      CORE_EXTENSIONS: 'icu;json;parquet;tpch'
      GEN: ninja

    steps:
    - uses: actions/checkout@v4

    - name: Install
      shell: bash
      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build valgrind clang

    - uses: ./.github/actions/ccache-action

    - name: Build
      shell: bash
      run: make relassert

    - name: Output version info
      shell: bash
      run: ./build/relassert/duckdb -c "PRAGMA version;"

    - name: Test
      shell: bash
      run: valgrind ./build/relassert/test/unittest test/sql/tpch/tpch_sf001.test_slow

 threadsan:
    name: Thread Sanitizer
    needs: linux-debug
    runs-on: ubuntu-24.04
    env:
      CC: clang
      CXX: clang++
      GEN: ninja
      BUILD_JEMALLOC: 1
      CORE_EXTENSIONS: "icu;tpch;tpcds;fts;json;inet"
      TSAN_OPTIONS: suppressions=${{ github.workspace }}/.sanitizer-thread-suppressions.txt
      DUCKDB_TEST_DESCRIPTION: 'Tests run with thread sanitizer.'

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - name: Install
      shell: bash
      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build clang

    - uses: ./.github/actions/ccache-action

    - name: Build
      shell: bash
      run: THREADSAN=1 make reldebug

    - name: Test
      shell: bash
      run: |
          python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest --no-exit --timeout 600
          python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest "[intraquery]" --no-exit --timeout 600
          python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest "[interquery]" --no-exit --timeout 1800
          python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest "[interquery]" --no-exit --timeout 1800 --force-storage
          python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest "[interquery]" --no-exit --timeout 1800 --force-storage --force-reload
          python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest "[detailed_profiler]" --no-exit --timeout 600
          python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest test/sql/tpch/tpch_sf01.test_slow --no-exit --timeout 600

 amalgamation-tests:
    name: Amalgamation Tests
    runs-on: ubuntu-22.04
    needs: linux-debug
    env:
      CC: clang
      CXX: clang++

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        ref: ${{ github.ref }}

    - uses: actions/setup-python@v5
      with:
        python-version: '3.12'

    - name: Install LLVM and Clang
      uses: KyleMayes/install-llvm-action@v1
      with:
        version: "14.0"

    - name: Generate Amalgamation
      shell: bash
      run:  |
          python scripts/amalgamation.py --extended
          clang++ -std=c++17 -Isrc/amalgamation src/amalgamation/duckdb.cpp -emit-llvm -S -O0

 # TODO: Consider bringing back fts
 # TODO: DEBUG_STACKTRACE: 1 + reldebug ?
 linux-configs:
    name: Tests a release build with different configurations
    runs-on: ubuntu-24.04
    needs: linux-debug
    env:
      BASE_BRANCH: ${{ github.base_ref || 'main' }}

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - uses: actions/setup-python@v5
      with:
        python-version: '3.12'

    - name: Install
      shell: bash
      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build libcurl4-openssl-dev

    - uses: ./.github/actions/ccache-action

    - name: Build
      id: build
      shell: bash
      env:
        CORE_EXTENSIONS: "json;parquet;icu;tpch;tpcds;autocomplete"
        GEN: ninja
      run: make

    - name: test/configs/force_storage.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/force_storage.json

    - name: test/configs/force_storage_restart.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/force_storage_restart.json

    - name: test/configs/latest_storage.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/latest_storage.json

    - name: test/configs/block_verification.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/block_verification.json

    - name: test/configs/verify_fetch_row.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/verify_fetch_row.json

    - name: test/configs/wal_verification.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/wal_verification.json

    - name: test/configs/vacuum_rebuild_indexes_force_storage.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/vacuum_rebuild_indexes_force_storage.json

    - name: test/configs/prefetch_all_parquet_files.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/prefetch_all_parquet_files.json

    - name: test/configs/no_local_filesystem.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/no_local_filesystem.json

    - name: test/configs/block_size_16kB.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/block_size_16kB.json

    - name: test/configs/latest_storage_block_size_16kB.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/latest_storage_block_size_16kB.json

    - name: test/configs/enable_verification.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/enable_verification.json

    - name: test/configs/block_allocator_100mib.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/block_allocator_100mib.json

    - name: Test dictionary_expression
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --verify-vector dictionary_expression --skip-compiled

    - name: Test dictionary_operator
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --verify-vector dictionary_operator --skip-compiled

    - name: Test constant_operator
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --verify-vector constant_operator --skip-compiled

    - name: Test sequence_operator
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --verify-vector sequence_operator --skip-compiled

    - name: Test nested_shuffle
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --verify-vector nested_shuffle --skip-compiled

    - name: Test variant_vector
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/variant_vector.json

    - name: Test variant_vector
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/compressed_in_memory.json

    - name: Test sycnronous table scan implementation
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --on-init "SET debug_physical_table_scan_execution_strategy=SYNCHRONOUS;"

    - name: Test block prefetching
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/prefetch_all_storage.json

    - name: Forwards compatibility tests
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        python3 scripts/test_storage_compatibility.py --versions "1.2.1|1.3.2|1.4.3" --new-unittest build/release/test/unittest


    # TODO: clean this up: we should probably be able to run this whole test suite with httpfs
    # We want to run the remainder of tests with httpfs
    - name: Setup vcpkg
      if: (success() || failure()) && steps.build.conclusion == 'success'
      uses: lukka/run-vcpkg@v11.1
      with:
        vcpkgGitCommitId: 84bab45d415d22042bd0b9081aea57f362da3f35

    - name: Build with httpfs extension
      id: build-httpfs
      shell: bash
      if: (success() || failure()) && steps.build.conclusion == 'success'
      env:
        CORE_EXTENSIONS: "json;parquet;icu;tpch;tpcds;httpfs"
        GEN: ninja
        VCPKG_TOOLCHAIN_PATH: ${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake
        USE_MERGED_VCPKG_MANIFEST: 1
      run:
        make clean && make

    - name: test/configs/encryption.json
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/encryption.json

    - name: Test PEG parser on fallback mode
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/peg_parser.json

    - name: Test PEG parser on strict when supported mode
      if: (success() || failure()) && steps.build.conclusion == 'success'
      shell: bash
      run: |
        ./build/release/test/unittest --test-config test/configs/peg_parser_strict.json