716 add build tests for macos and windows to ci pipeline (#764)

* renamed ci to build workflows

* ci_pipeline - added new jobs with reusable workflows with build tests on windows, macos and ubuntu arm

* ci_pipeline - added new jobs with reusable workflows with build tests on windows, macos and ubuntu arm

* switching from reusable workflows to local, added build and tests for Windows, MacOS and Ubuntu on Arm to the ci_workflow

* commented out arena_test in the windows workflow

* Renamed some jobs

* Renamed some jobs

* commented out the test for Windows, leaving build only

* re-arranges the ci workflow and added a summary job which will be made as required check

* updated condition of build and other jobs that come after check license

* updated condition of build and other jobs that come after check license

* updated condition of build and other jobs that come after check license

* fix instructions for macos_Arm

* upgraded gh cache action from v2 to v3.3.2

* merged the updated check license and history workflow and disabled condition for it execution from ci_pipeline that made sure it is running only on PR review

* switched github.ref_name to github.ref for Linux on Arm build

* added a test that checks the workflow trigger and sets up the correct branch for building on Linux Arm, this test is needed for docker container that we use for the build

* added Dump GitHub context step to Linux-Arm-build

* added case pull_request_review for build on Linux Arm

* added case pull_request_review for build on Linux Arm

* added case pull_request_review for build on Linux Arm

---------

Co-authored-by: speedbadmin <[email protected]>

Author: maxb-io

.github/workflows/build_and_publish_jar.yml

@@ -30,12 +30,13 @@ jobs:
       run: |
         major=$(grep '_MAJOR\s\+[0-9]\+' "$VERSION_FILE" | sed 's/[^0-9]\+//') && echo $major
         minor=$(grep '_MINOR\s\+[0-9]\+' "$VERSION_FILE" | sed 's/[^0-9]\+//') && echo $minor
-        patch=$(( $(grep '_PATCH\s\+[0-9]\+' "$VERSION_FILE" | sed 's/[^0-9]\+//') + 1 )) && echo $patch
+        #patch=$(( $(grep '_PATCH\s\+[0-9]\+' "$VERSION_FILE" | sed 's/[^0-9]\+//') + 1 )) && echo $patch
+        patch=$(( $(grep '_PATCH\s\+[0-9]\+' "$VERSION_FILE" | sed 's/[^0-9]\+//'))) && echo $patch
         echo "verSion=$major.$minor.$patch"  >> $GITHUB_OUTPUT
        
   Mac_i86:
     needs: pre_build
-    uses: ./.github/workflows/ci_macos.yml
+    uses: ./.github/workflows/build_macos.yml
     with:
       verSion: ${{ needs.pre_build.outputs.out1 }}     
     secrets:
@@ -45,19 +46,19 @@ jobs:
 
   Mac_ARM:
     needs: pre_build
-    uses: ./.github/workflows/ci_macos_ARM.yml
+    uses: ./.github/workflows/build_macos_ARM.yml
     with:
       verSion: ${{ needs.pre_build.outputs.out1 }}  
 
   Ubuntu_ARM:
     needs: pre_build
-    uses: ./.github/workflows/ci_ubuntu_arm.yml
+    uses: ./.github/workflows/build_ubuntu_arm.yml
     with:
       verSion: ${{ needs.pre_build.outputs.out1 }}  
 
   Windows:
     needs: pre_build
-    uses: ./.github/workflows/ci_windows.yml
+    uses: ./.github/workflows/build_windows.yml
     with:
       verSion: ${{ needs.pre_build.outputs.out1 }}
     secrets:

.github/workflows/ci_macos.yml renamed to .github/workflows/build_macos.yml

@@ -26,12 +26,12 @@ jobs:
         echo "the built library java/target/libspeedbjni-osx-x86_64.jnilib needs to be uploaded to the folloing location"
         echo "java/target/libspeedbjni-osx-arm64.jnilib s3://spdb-builder/jar_test/v${{ inputs.verSion }}/libspeedbjni-osx-arm64.jnilib"
         
-        # export JAVA_HOME=/usr/local/opt/openjdk@11
-        # export CPPFLAGS="-I/usr/local/opt/openjdk@11/include"
-        # export CXXFLAGS="-I/usr/local/opt/openjdk@11/include"
+        # export JAVA_HOME=`/usr/libexec/java_home -v 11`
+        # export CPPFLAGS="-I$JAVA_HOME/include"
+        # export CXXFLAGS="-I$JAVA_HOME/include"
         # ROCKSDB_DISABLE_JEMALLOC=1 PORTABLE=1 DEBUG_LEVEL=0 make -j 4 rocksdbjavastatic
     
     - name: 'upload artifacts'    #This step executed only when this workflow is called by another and a version is provided
       if: inputs.verSion != ' '
       run: echo "nothing to do here until the real environment is ready"
-      #run: aws s3 cp java/target/libspeedbjni-osx-x86_64.jnilib s3://spdb-builder/jar_test/v${{ inputs.verSion }}/libspeedbjni-osx-x86_64.jnilib
+      #run: aws s3 cp java/target/libspeedbjni-osx-x86_64.jnilib s3://spdb-builder/jar_test/v${{ inputs.verSion }}/libspeedbjni-osx-arm64.jnilib

.github/workflows/ci_macos_ARM.yml renamed to .github/workflows/build_macos_ARM.yml

@@ -13,10 +13,14 @@ permissions: write-all
 jobs:
   #Sanity:
     #uses: speedb-io/speedb/.github/workflows/sanity_check.yml@main
+
+  Check-Licence-And-History:
+    #if: ${{  github.event_name == 'pull_request_review' }}
+    uses: ./.github/workflows/check_license_and_history.yml
 
   Build:
-    #needs: [Sanity]
-    if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
+    needs: [Check-Licence-And-History]
+    if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }}
     #runs-on: [self-hosted, ubuntu, asrunner]
     runs-on: ubuntu-20.04
     strategy:
@@ -58,7 +62,7 @@ jobs:
         
         
     - name: ccache cache files
-      uses: actions/cache@v2
+      uses: actions/cache@v3.3.2
       with:
         path: ~/.ccache
         key: ${{runner.os}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
@@ -104,15 +108,269 @@ jobs:
     #uses: speedb-io/speedb/.github/workflows/perf-test.yml@main
 
   QA-Tests:
-    if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
+    if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }}
     needs: [Build]
     uses: speedb-io/speedb/.github/workflows/qa-tests.yml@main
 
   Fuzz:
-    if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
+    if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }}
     needs: [Build]
     uses: ./.github/workflows/test_fuzz.yml
 
-  Check-Licence-And-History:
-    if: ${{  github.event_name == 'pull_request_review' }}
-    uses: ./.github/workflows/check_license_and_history.yml
+  Windows-build-test:
+    if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }}
+    needs: [Build]
+    runs-on: windows-2022
+    env: 
+      THIRDPARTY_HOME: C:/Users/runneradmin/thirdparty
+      CMAKE_HOME: C:/Program Files/CMake
+      CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
+      SNAPPY_HOME: C:/Users/runneradmin/thirdparty/snappy-1.1.9
+      SNAPPY_INCLUDE: C:/Users/runneradmin/thirdparty/snappy-1.1.9;C:/Users/circleci/thirdparty/snappy-1.1.9/build
+      SNAPPY_LIB_DEBUG: C:/Users/runneradmin/thirdparty/snappy-1.1.9/build/Debug/snappy.lib
+      CMAKE_GENERATOR: Visual Studio 17 2022
+      CODE_HOME: C:/Users/runneradmin/code
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin' # See 'Supported distributions' for available options
+          java-version: '8'
+
+      - name: Add msbuild to PATH
+        uses: microsoft/[email protected]    
+
+      - name: Setup VS Dev
+        uses: seanmiddleditch/gha-setup-vsdevenv@v4
+
+      - name: install cmake 
+        shell: powershell
+        run: |                  
+            echo "Installing CMake..."
+            choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' -y
+      - name: prepare Thirdparty home
+        shell: powershell
+        run: mkdir "$Env:THIRDPARTY_HOME"
+
+      - name: install snappy test
+        shell: powershell
+        run: |
+            mkdir $env:CODE_HOME
+            cd $env:CODE_HOME
+            curl https://github.com/google/snappy/archive/refs/tags/1.1.9.zip -o 1.1.9.zip
+            Expand-Archive -Path 1.1.9.zip -DestinationPath snappy-tmp
+            mv .\snappy-tmp\snappy-1.1.9\ .
+            rmdir .\snappy-tmp\
+            cd .\snappy-1.1.9\
+            mkdir build
+            cd .\build\
+            cmake -G "Visual Studio 17 2022" -A x64 -DCMAKE_GENERATOR_PLATFORM=x64 -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF ..
+            msbuild Snappy.sln /p:Configuration=Debug /p:Platform=x64
+            msbuild Snappy.sln /p:Configuration=Release /p:Platform=x64        
+            
+      - name: install snappy
+        shell: powershell
+        run: |      
+          cd $Env:THIRDPARTY_HOME
+          curl https://github.com/google/snappy/archive/refs/tags/1.1.9.zip -O snappy-1.1.9.zip
+          Expand-Archive -Path snappy-1.1.9.zip -DestinationPath snappy-tmp
+          mv .\snappy-tmp\snappy-1.1.9\ .
+          cd snappy-1.1.9
+          mkdir build
+          cd .\build
+          & cmake -G "Visual Studio 17 2022" -A x64 -DCMAKE_GENERATOR_PLATFORM=x64 -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF ..                        
+          msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Release -property:Platform=x64
+          
+      - name: install gflags
+        shell: powershell
+        run: |
+            cd $Env:THIRDPARTY_HOME
+            curl https://github.com/gflags/gflags/archive/refs/tags/v2.2.2.zip -o v2.2.2.zip
+            Expand-Archive -Path .\v2.2.2.zip -DestinationPath gflags-tmp      
+            mv .\gflags-tmp\gflags-2.2.2 .
+            rmdir gflags-tmp
+            cd gflags-2.2.2
+            mkdir target
+            cd target
+            cmake -G "Visual Studio 17 2022" -A x64 ..
+            msbuild gflags.sln /p:Configuration=Release /p:Platform=x64
+      - name: install zlib
+        shell: powershell
+        run: |    
+            cd $Env:THIRDPARTY_HOME
+            curl   https://zlib.net/zlib13.zip -o  zlib13.zip
+            Expand-Archive -Path zlib13.zip -DestinationPath zlib-tmp
+            mv .\zlib-tmp\zlib-1.3\ .
+            rmdir zlib-tmp
+            cd zlib-1.3\contrib\vstudio\vc14
+            devenv zlibvc.sln /upgrade
+            cp ../../../zlib.h .
+            msbuild zlibvc.sln /p:Configuration=Debug /p:Platform=x64
+            msbuild zlibvc.sln /p:Configuration=Release /p:Platform=x64
+            copy x64\ZlibDllRelease\zlibwapi.lib x64\ZlibStatRelease\
+      - name: install lz4
+        shell: powershell
+        run: |    
+            cd $Env:THIRDPARTY_HOME
+            curl https://github.com/lz4/lz4/archive/refs/tags/v1.9.2.zip -o lz4.zip
+            Expand-Archive -Path lz4.zip -DestinationPath lz4-tmp
+            mv .\lz4-tmp\lz4-1.9.2\ .
+            rmdir .\lz4-tmp\
+            cd .\lz4-1.9.2\
+            cd visual\VS2017
+            devenv lz4.sln /upgrade
+            msbuild lz4.sln /p:Configuration=Release /p:Platform=x64
+      - name: install zctd
+        shell: powershell
+        run: |
+            cd $Env:THIRDPARTY_HOME
+            curl   https://github.com/facebook/zstd/archive/v1.5.2.zip -o zstd-tmp.zip
+            Expand-Archive -Path zstd-tmp.zip -DestinationPath zstd-tmp
+            mv .\zstd-tmp\zstd-1.5.2\ .
+            rmdir .\zstd-tmp\
+            cd zstd-1.5.2\build\VS2010
+            devenv zstd.sln /upgrade
+            msbuild zstd.sln /p:Configuration=Debug /p:Platform=x64
+            msbuild zstd.sln /p:Configuration=Release /p:Platform=x64
+    
+      - name: Build Speedb      
+        run: |
+            copy C:\Users\runneradmin\thirdparty\snappy-1.1.9\build\snappy-stubs-public.h C:\Users\runneradmin\thirdparty\snappy-1.1.9\
+            copy tools\thirdparty.txt thirdparty.inc # copy the thirdparty.inc that reflects the env on the runner machine
+            mkdir build
+            cd build
+            & $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=20 -A x64 -DJNI=1 -DGFLAGS=1 -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 -DSNAPPY=1 -DLZ4=1 -DZLIB=1 -DZSTD=1 -DXPRESS=1 -DFAIL_ON_WARNINGS=0 ..
+            cd ..
+            echo "Building with VS version: $Env:CMAKE_GENERATOR"
+            msbuild build/speedb.sln /p:Configuration=Release /t:speedbjni-shared
+            #msbuild.exe build/speedb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
+    
+      - name: Test Speedb
+        shell: powershell
+        run: |
+          echo "skipping all tests for now"          
+          #build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
+          #build_tools\run_ci_db_test.ps1 -SuiteRun db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
+
+    
+
+  Macos-build:
+    if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }}
+    needs: [Build]
+    runs-on: macos-11
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin' # See 'Supported distributions' for available options
+          java-version: '8'
+      - name: build jar
+        run: |
+          echo $JAVA_HOME
+          export CPPFLAGS="-I$JAVA_HOME/include"
+          export CXXFLAGS="-I$JAVA_HOME/include"
+          brew install zlib
+          brew install bzip2 lz4 snappy
+          ROCKSDB_DISABLE_JEMALLOC=1 PORTABLE=1 DEBUG_LEVEL=0 make -j 4 rocksdbjavastatic
+
+
+  Linux-build:
+    if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }}
+    needs: [Build]
+    runs-on: ubuntu-latest
+
+    container:
+      image: centos:7.9.2009
+
+    steps:
+      - name: pre
+        run: |
+          yum install -y centos-release-scl epel-release
+          yum install -y make devtoolset-11-gcc-c++ \
+            coreutils wget unzip which git python3 openssl openssl-devel \
+            libzstd-devel lz4-devel snappy-devel zlib-devel readline-devel \
+            java-1.8.0-openjdk-devel
+          echo "PATH=/opt/rh/devtoolset-11/root/usr/bin:${PATH}" >> $GITHUB_ENV
+          echo "RELEASE_VERSION=${GITHUB_REF_NAME#speedb/v}" >> $GITHUB_ENV
+
+      - name: Install CMake
+        run: |
+          CMAKE_RELEASE=3.20.1
+          wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_RELEASE}/cmake-${CMAKE_RELEASE}.tar.gz
+          tar xf cmake-${CMAKE_RELEASE}.tar.gz
+          cd cmake-${CMAKE_RELEASE}
+          ./bootstrap
+          make -j$(nproc) && make install
+
+      - uses: actions/checkout@v3
+
+      - run: mkdir "$GITHUB_WORKSPACE/out"
+
+      - name: Build and package release libraries
+        run: |
+          rm -rf build && mkdir build && cd build
+          cmake .. -DCMAKE_BUILD_TYPE=Release -DSPDB_RELEASE_BUILD=1 -DPORTABLE=1 -DWITH_GFLAGS=0 -DWITH_SNAPPY=1 -DWITH_LZ4=1 -DWITH_ZLIB=1 -DWITH_ZSTD=1
+          
+      - name: Build Jar
+        run: |
+          make clean
+          SPDB_RELEASE_BUILD=1 LIB_MODE=static DEBUG_LEVEL=0 PORTABLE=1 JAVA_HOME=/usr/lib/jvm/java-openjdk make -j$(nproc) rocksdbjavastatic
+         
+      - name: Build db_bench
+        run: |
+          yum install -y gflags-devel
+          rm -rf build && mkdir build && cd build
+          cmake .. -DCMAKE_BUILD_TYPE=Release -DSPDB_RELEASE_BUILD=1 -DPORTABLE=1 -DWITH_GFLAGS=1 \
+            -DWITH_SNAPPY=1 -DWITH_LZ4=1 -DWITH_ZLIB=1 -DWITH_ZSTD=1 \
+            -DWITH_BENCHMARK_TOOLS=1 -DROCKSDB_BUILD_SHARED=1
+          make -j$(nproc) db_bench
+
+
+  Linux-Arm-build:
+    if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }}
+    needs: [Build]
+    runs-on: ubuArm64G 
+    steps:
+    - name: Dump GitHub context
+      env:
+        GITHUB_CONTEXT: ${{ toJson(github) }}
+      run: |
+        echo "$GITHUB_CONTEXT"  
+    
+    - name: 'Checkout GitHub Action'
+      uses: actions/checkout@v3
+
+    - name: 'Build_on_Arm_Centos'
+      run: |
+        export SPDB_LIB_DIR=~/spdb_lib &&  mkdir -p $SPDB_LIB_DIR
+        case "$GITHUB_EVENT_NAME" in 
+          "pull_request")  
+            echo "this workflow was triggered by a pull request"
+            echo "the branch can not be used, it is $GITHUB_REF_NAME , instead ${{ github.head_ref }} will be used"
+            docker run --rm -v $(readlink -f ${SPDB_LIB_DIR}):/out -i speedb-centos-builder ${{ github.head_ref }}
+            ;;
+          "workflow_dispatch") 
+            echo "this workflow was triggered by a workflow dispatch, we will use the ref_name instead of the merge branch"
+            echo " consider using github.ref, the branch that will be used here: " ${{ github.ref_name }}
+            docker run --rm -v $(readlink -f ${SPDB_LIB_DIR}):/out -i speedb-centos-builder ${{ github.ref_name }}
+            ;;
+          "pull_request_review")  
+            echo "this workflow was triggered by a pull request review"
+            echo "the branch can not be used, it is $GITHUB_REF_NAME , instead  ${{ github.event.pull_request.head.ref }} will be used"
+            docker run --rm -v $(readlink -f ${SPDB_LIB_DIR}):/out -i speedb-centos-builder ${{ github.event.pull_request.head.ref }}
+            ;;
+        esac   
+   
+  CI-all:
+    if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
+    needs: [Check-Licence-And-History, Build, QA-Tests, Fuzz, Linux-Arm-build, Linux-build, Macos-build, Windows-build-test]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Summary
+      run: |
+        echo "All tests passed"
+        exit 0