[POC] Support leading segment merge with PQ #1813
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test k-NN | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' # every night | |
| push: | |
| branches: | |
| - "*" | |
| - "feature/**" | |
| paths: | |
| - 'build.gradle' | |
| - 'settings.gradle' | |
| - 'src/**' | |
| - 'build-tools/**' | |
| - 'buildSrc/**' | |
| - 'gradle/**' | |
| - '.github/workflows/CI.yml' | |
| pull_request: | |
| branches: | |
| - "*" | |
| - "feature/**" | |
| paths: | |
| - 'build.gradle' | |
| - 'settings.gradle' | |
| - 'src/**' | |
| - 'build-tools/**' | |
| - 'buildSrc/**' | |
| - 'gradle/**' | |
| - '.github/workflows/CI.yml' | |
| jobs: | |
| Get-CI-Image-Tag: | |
| uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main | |
| with: | |
| product: opensearch | |
| Build-k-NN-Linux: | |
| strategy: | |
| matrix: | |
| java: [21, 25] | |
| env: | |
| CC: gcc10-gcc | |
| CXX: gcc10-g++ | |
| FC: gcc10-gfortran | |
| name: Build and Test jVector k-NN Plugin on Linux | |
| runs-on: ubuntu-latest | |
| needs: Get-CI-Image-Tag | |
| container: | |
| # using the same image which is used by opensearch-build team to build the OpenSearch Distribution | |
| # this image tag is subject to change as more dependencies and updates will arrive over time | |
| image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} | |
| # need to switch to root so that github actions can install runner binary on container without permission issues. | |
| options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} | |
| steps: | |
| - name: Run start commands | |
| run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} | |
| - name: Checkout jVector k-NN | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Setup Java ${{ matrix.java }} | |
| uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| - name: Run build | |
| # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. | |
| run: | | |
| ulimit -n 65536 | |
| chown -R 1000:1000 `pwd` | |
| if lscpu | grep -i avx512f | grep -i avx512cd | grep -i avx512vl | grep -i avx512dq | grep -i avx512bw | |
| then | |
| if lscpu | grep -q "GenuineIntel" && lscpu | grep -i avx512_fp16 | grep -i avx512_bf16 | grep -i avx512_vpopcntdq | |
| then | |
| echo "the system is an Intel(R) Sapphire Rapids or a newer-generation processor" | |
| su `id -un 1000` -c "whoami && java -version && ./gradlew build -Davx512_spr.enabled=true -Dnproc.count=`nproc`" | |
| else | |
| echo "avx512 available on system" | |
| su `id -un 1000` -c "whoami && java -version && ./gradlew build -Davx512.enabled=true -Dnproc.count=`nproc`" | |
| fi | |
| elif lscpu | grep -i avx2 | |
| then | |
| echo "avx2 available on system" | |
| su `id -un 1000` -c "whoami && java -version && ./gradlew build -Davx512.enabled=false -Davx512_spr.enabled=false -Dnproc.count=`nproc`" | |
| else | |
| echo "avx512 and avx2 not available on system" | |
| su `id -un 1000` -c "whoami && java -version && ./gradlew build -Davx2.enabled=false -Davx512.enabled=false -Davx512_spr.enabled=false -Dnproc.count=`nproc`" | |
| fi | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| if: always() | |
| with: | |
| name: ${{ matrix.java }}-linux-reports | |
| path: | | |
| ./build/reports/ | |
| ./build/testclusters/*/logs | |
| - name: Upload Coverage Report | |
| uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| Build-k-NN-Linux-Native: | |
| strategy: | |
| matrix: | |
| java: [21, 25] | |
| name: Build and Test jVector k-NN Plugin on Linux (Native) | |
| runs-on: ubuntu-latest | |
| needs: Get-CI-Image-Tag | |
| container: | |
| # using the same image which is used by opensearch-build team to build the OpenSearch Distribution | |
| # this image tag is subject to change as more dependencies and updates will arrive over time | |
| image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} | |
| # need to switch to root so that github actions can install runner binary on container without permission issues. | |
| options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} | |
| steps: | |
| - name: Run start commands | |
| run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} | |
| - name: Checkout jVector k-NN | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Setup Java ${{ matrix.java }} | |
| uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| - name: Run build | |
| # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. | |
| run: | | |
| if lscpu | grep -i avx512f | grep -i avx512cd | grep -i avx512vl | grep -i avx512dq | grep -i avx512bw; then | |
| echo "avx512 available on system" | |
| ulimit -n 65536 | |
| chown -R 1000:1000 `pwd` | |
| su `id -un 1000` -c "whoami && java -version && ./gradlew build -Davx512.enabled=true -Davx512_spr.enabled=true -Dnproc.count=`nproc` -Djvector.experimental.enable_native_vectorization=true" | |
| else | |
| echo "avx512 not available on system" | |
| exit 0 | |
| fi | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| if: always() | |
| with: | |
| name: ${{ matrix.java }}-linux-native-reports | |
| path: | | |
| ./build/reports/ | |
| ./build/testclusters/*/logs | |
| - name: Upload Coverage Report | |
| uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| Build-k-NN-MacOS: | |
| strategy: | |
| matrix: | |
| java: [21, 25] | |
| name: Build and Test jVector k-NN Plugin on MacOS | |
| needs: Get-CI-Image-Tag | |
| runs-on: macos-latest | |
| steps: | |
| - name: Checkout jVector k-NN | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Setup Java ${{ matrix.java }} | |
| uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| - name: Run build | |
| run: | | |
| ./gradlew build | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| if: always() | |
| with: | |
| name: ${{ matrix.java }}-macos-reports | |
| path: | | |
| ./build/reports/ | |
| ./build/testclusters/*/logs | |
| Build-k-NN-Windows: | |
| strategy: | |
| matrix: | |
| java: [21, 25] | |
| name: Build and Test jVector k-NN Plugin on Windows | |
| needs: Get-CI-Image-Tag | |
| runs-on: windows-latest | |
| steps: | |
| - name: Checkout jVector k-NN | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Setup Java ${{ matrix.java }} | |
| uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| distribution: 'temurin' | |
| - name: Install MinGW Using Scoop | |
| run: | | |
| iex "& {$(irm get.scoop.sh)} -RunAsAdmin" | |
| scoop bucket add main | |
| scoop install mingw | |
| - name: Add MinGW to PATH | |
| run: | | |
| echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH | |
| Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" | |
| refreshenv | |
| - name: Run build | |
| run: | | |
| ./gradlew.bat build | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| if: always() | |
| with: | |
| name: ${{ matrix.java }}-windows-reports | |
| path: | | |
| ./build/reports/ | |
| ./build/testclusters/*/logs |