ci-pr-validation.yaml

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

name: PR validation
on:
  pull_request:
    branches:
      - main

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:

  formatting-check:
    name: Formatting Check
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    - name: Run clang-format style check for C/C++/Protobuf programs.
      uses: jidicula/clang-format-action@v4.11.0
      with:
        clang-format-version: '11'
        exclude-regex: '.*\.(proto|hpp)'

  wireshark-dissector-build:
    name: Build the Wireshark dissector
    needs: formatting-check
    runs-on: ${{ matrix.os }}
    timeout-minutes: 60
    strategy:
      matrix:
        # TODO: support build on macos-14
        os: [ubuntu-latest]

    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Install deps (Ubuntu)
        if: ${{ startsWith(matrix.os, 'ubuntu') }}
        run: |
          sudo apt-get update -y
          sudo apt-get install -y protobuf-compiler libprotobuf-dev wireshark-dev

      - name: Install deps (macOS)
        if: ${{ startsWith(matrix.os, 'macos') }}
        run: |
          # See https://github.com/Homebrew/homebrew-core/issues/157142
          export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
          cd /usr/local/bin
          rm -f 2to3* idle3* pydoc* python3*
          rm -f /usr/local/share/man/man1/python3.1 /usr/local/lib/pkgconfig/python3*
          cd /usr/local/Frameworks/Python.framework
          rm -rf Headers Python Resources Versions/Current
          brew update
          brew install pkg-config wireshark protobuf
      - name: Build wireshark plugin
        run: |
          cmake -S wireshark -B build-wireshark
          cmake --build build-wireshark

  lint:
    name: Lint
    needs: formatting-check
    runs-on: ubuntu-24.04
    timeout-minutes: 120

    steps:
      - name: checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          submodules: recursive

      - name: Restore vcpkg installed cache
        uses: actions/cache@v4
        with:
          path: build/vcpkg_installed
          key: vcpkg-${{ runner.os }}-${{ hashFiles('vcpkg.json', 'CMakeLists.txt', 'vcpkg-triplets/**') }}
          restore-keys: vcpkg-${{ runner.os }}-

      - name: Build the project
        run: |
          cmake -B build -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
          cmake --build build -j8

      - name: Tidy check
        run: |
          sudo apt-get install -y clang-tidy
          ./build-support/run_clang_tidy.sh
          if [[ $? -ne 0 ]]; then
              echo "clang-tidy failed"
              exit 1
          fi

  unit-tests:
    name: Run unit tests
    needs: formatting-check
    runs-on: ubuntu-24.04
    timeout-minutes: 120

    steps:
      - name: checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          submodules: recursive

      - name: Restore vcpkg installed cache
        uses: actions/cache@v4
        with:
          path: |
            build/vcpkg_installed
            build-boost-asio/vcpkg_installed
          key: vcpkg-${{ runner.os }}-${{ hashFiles('vcpkg.json', 'CMakeLists.txt', 'vcpkg-triplets/**') }}
          restore-keys: vcpkg-${{ runner.os }}-

      - name: Build core libraries
        run: |
          cmake -B build -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=OFF
          cmake --build build -j8

      - name: Check formatting
        run: |
          ./vcpkg/vcpkg format-manifest vcpkg.json
          if [[ $(git diff | wc -l) -gt 0 ]]; then
              echo "Please run `./vcpkg/vcpkg format-manifest vcpkg.json` to reformat vcpkg.json"
              exit 1
          fi

      - name: Build tests
        run: |
          cmake -B build -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
          cmake --build build -j8

      - name: Install gtest-parallel
        run: |
          sudo curl -o /gtest-parallel https://raw.githubusercontent.com/google/gtest-parallel/master/gtest_parallel.py 

      - name: Run unit tests
        run: RETRY_FAILED=3 CMAKE_BUILD_DIRECTORY=./build ./run-unit-tests.sh

      - name: Build with Boost.Asio
        run: |
          cmake -B build-boost-asio -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=ON
          cmake --build build-boost-asio -j8

      - name: Build perf tools
        run: |
          cmake -B build -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=ON -DBUILD_PERF_TOOLS=ON
          cmake --build build -j8

      - name: Verify custom vcpkg installation
        run: |
          mv vcpkg /tmp/vcpkg-custom
          cmake -B build-2 -DINTEGRATE_VCPKG=ON -DCMAKE_TOOLCHAIN_FILE="/tmp/vcpkg-custom/scripts/buildsystems/vcpkg.cmake"

  cpp20-build:
    name: Build with the C++20 standard
    needs: lint
    runs-on: ubuntu-22.04
    timeout-minutes: 60

    steps:
      - name: checkout
        uses: actions/checkout@v3
      - name: Install deps
        run: |
          sudo apt-get update -y
          sudo apt-get install -y libcurl4-openssl-dev libssl-dev \
            protobuf-compiler libprotobuf-dev libboost-dev \
            libboost-dev libboost-program-options-dev \
            libzstd-dev libsnappy-dev libgmock-dev libgtest-dev
      - name: CMake
        run: cmake -B build -DBUILD_PERF_TOOLS=ON -DCMAKE_CXX_STANDARD=20
      - name: Build
        run: |
          cmake --build build -j8 --target pulsarShared pulsarStatic
          cmake --build build -j8

  cpp-build-windows:
    timeout-minutes: 120
    name: Build CPP Client on ${{ matrix.name }}
    needs: unit-tests
    runs-on: ${{ matrix.os }}
    env:
      VCPKG_ROOT: '${{ github.workspace }}/vcpkg'
      INSTALL_DIR: 'C:\\pulsar-cpp'
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: 'Windows x64'
            os: windows-2022
            triplet: x64-windows-static
            suffix: 'windows-win64'
            generator: 'Visual Studio 17 2022'
            arch: '-A x64'
          - name: 'Windows x86'
            os: windows-2022
            triplet: x86-windows-static
            suffix: 'windows-win32'
            generator: 'Visual Studio 17 2022'
            arch: '-A Win32'

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true

      - name: Restore vcpkg and its artifacts.
        uses: actions/cache@v4
        id: vcpkg-cache
        continue-on-error: true
        with:
          path: |
            ${{ github.workspace }}/vcpkg_installed
            ${{ env.VCPKG_ROOT }}/downloads
            ${{ env.VCPKG_ROOT }}/vcpkg.exe
          key: ${{ runner.os }}-${{ matrix.triplet }}-vcpkg-${{ hashFiles('vcpkg.json') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.triplet }}-vcpkg-
          save-always: true

      - name: Get vcpkg(windows)
        if: ${{ runner.os == 'Windows' && steps.vcpkg-cache.outputs.cache-hit != 'true' }}
        shell: pwsh
        run: |
          cd ${{ github.workspace }}
          mkdir build -Force
          # If vcpkg.exe already exists, skip bootstrapping to save time and avoid
          # modifying the checked-out vcpkg directory.
          if (Test-Path "${{ github.workspace }}\\vcpkg\\vcpkg.exe") {
            Write-Host "vcpkg.exe present — skipping bootstrap"
          } else {
            if (Test-Path vcpkg) {
              # If vcpkg was checked out as a submodule it may be shallow; fetch full history
              git -C vcpkg fetch --unshallow || true
            } else {
              git clone https://github.com/Microsoft/vcpkg.git
            }
            Push-Location vcpkg
            .\bootstrap-vcpkg.bat
            Pop-Location
          }

      - name: remove system vcpkg(windows)
        if: runner.os == 'Windows'
        run: rm -rf "$VCPKG_INSTALLATION_ROOT"
        shell: bash

      - name: Install vcpkg packages
        run: |
          ${{ env.VCPKG_ROOT }}\vcpkg.exe install --triplet ${{ matrix.triplet }}

      - name: Configure
        shell: bash
        run: |
          if [ "$RUNNER_OS" == "Windows" ]; then
            cmake \
              -B ./build-1 \
              -G "${{ matrix.generator }}" ${{ matrix.arch }} \
              -DUSE_ASIO=ON \
              -DBUILD_TESTS=OFF \
              -DVCPKG_TRIPLET="${{ matrix.triplet }}" \
              -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" \
              -S .
          fi

      - name: Install
        shell: bash
        run: |
          if [ "$RUNNER_OS" == "Windows" ]; then
            cmake --build ./build-1 --parallel --config Release
            cmake --install ./build-1
          fi

      - name: Test examples
        shell: bash
        run: |
          if [ "$RUNNER_OS" == "Windows" ]; then
            cd win-examples
            cmake \
              -G "${{ matrix.generator }}" ${{ matrix.arch }} \
              -DLINK_STATIC=OFF \
              -DCMAKE_PREFIX_PATH=${{ env.INSTALL_DIR }} \
              -B build-dynamic
            cmake --build build-dynamic --config Release
            cmake \
              -G "${{ matrix.generator }}" ${{ matrix.arch }} \
              -DLINK_STATIC=ON \
              -DCMAKE_PREFIX_PATH=${{ env.INSTALL_DIR }} \
              -B build-static
            cmake --build build-static --config Release
            ./build-static/Release/win-example.exe
          fi

      - name: Build (Debug)
        shell: bash
        run: |
          if [ "$RUNNER_OS" == "Windows" ]; then
            cmake \
              -B ./build-2 \
              -G "${{ matrix.generator }}" ${{ matrix.arch }} \
              -DUSE_ASIO=ON \
              -DBUILD_TESTS=OFF \
              -DVCPKG_TRIPLET="${{ matrix.triplet }}" \
              -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" \
              -DCMAKE_BUILD_TYPE=Debug \
              -S .
            cmake --build ./build-2 --parallel --config Debug
          fi

  package:
    name: Build ${{matrix.pkg.name}} ${{matrix.cpu.platform}}
    runs-on: ubuntu-22.04
    needs: [lint, unit-tests]
    timeout-minutes: 500

    strategy:
      fail-fast: true
      matrix:
        pkg:
          - { name: 'RPM', type: 'rpm', path: 'pkg/rpm/RPMS' }
          - { name: 'Deb', type: 'deb', path: 'pkg/deb/BUILD/DEB' }
          - { name: 'Alpine', type: 'apk', path: 'pkg/apk/build' }
        cpu:
          - { arch: 'x86_64', platform: 'x86_64' }

    steps:
      - name: checkout
        uses: actions/checkout@v3

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2

      - name: Package Pulsar source
        run: build-support/generate-source-archive.sh

      - uses: docker/setup-buildx-action@v2

      - name: Build dependencies Docker image
        uses: docker/build-push-action@v3
        with:
          context: ./pkg/${{matrix.pkg.type}}
          load: true
          tags: build:latest
          platforms: linux/${{matrix.cpu.platform}}
          build-args: PLATFORM=${{matrix.cpu.arch}}
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - name: Build packages
        run: pkg/${{matrix.pkg.type}}/docker-build-${{matrix.pkg.type}}-${{matrix.cpu.platform}}.sh build:latest

      # TODO: verify the pre-built package works without linking issue

  cpp-build-macos:
    timeout-minutes: 120
    name: Build CPP Client on macOS with static dependencies
    runs-on: macos-14
    needs: lint
    steps:
      - name: checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          submodules: recursive

      - name: Restore vcpkg installed cache
        uses: actions/cache@v4
        with:
          path: build-osx/vcpkg_installed
          key: vcpkg-${{ runner.os }}-arm64-${{ hashFiles('vcpkg.json', 'CMakeLists.txt', 'vcpkg-triplets/**') }}
          restore-keys: vcpkg-${{ runner.os }}-arm64-

      - name: Build libraries
        run: ./pkg/mac/build-static-library.sh

  # Job that will be required to complete and depends on all the other jobs
  check-completion:
    name: Check Completion
    runs-on: ubuntu-latest
    needs: [formatting-check, wireshark-dissector-build, lint, unit-tests, cpp20-build, cpp-build-windows, package, cpp-build-macos]

    steps:
      - run: true