Add ci for milvus gpu (#22845)

Signed-off-by: Bennu-Li <yunmei.li@zilliz.com>

Author: Bennu

.env

@@ -3,6 +3,8 @@ IMAGE_ARCH=amd64
 OS_NAME=ubuntu20.04
 DATE_VERSION=20230215-5351fee
 LATEST_DATE_VERSION=20230215-5351fee
+GPU_DATE_VERSION=20230317-a1c7b0c
+LATEST_GPU_DATE_VERSION=20230317-a1c7b0c
 MINIO_ADDRESS=minio:9000
 PULSAR_ADDRESS=pulsar://pulsar:6650
-ETCD_ENDPOINTS=etcd:2379
+ETCD_ENDPOINTS=etcd:2379

.github/workflows/publish-builder.yaml

@@ -6,13 +6,13 @@ on:
   push:
     # file paths to consider in the event. Optional; defaults to all.
     paths:
-      - 'build/docker/builder/**'
+      - 'build/docker/builder/cpu/**'
       - '.github/workflows/publish-builder.yaml'
       - '!**.md'
   pull_request:
     # file paths to consider in the event. Optional; defaults to all.
     paths:
-      - 'build/docker/builder/**'
+      - 'build/docker/builder/cpu/**'
       - '.github/workflows/publish-builder.yaml'
       - '!**.md'
 

Makefile

@@ -39,8 +39,8 @@ milvus-gpu: build-cpp-gpu print-build-info
 	@echo "Building Milvus-gpu ..."
 	@source $(PWD)/scripts/setenv.sh && \
 		mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
-		GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
-		${AARCH64_FLAG} -o $(INSTALL_PATH)/milvus-gpu $(PWD)/cmd/main.go 1>/dev/null
+		GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS_GPU)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
+		${AARCH64_FLAG} -o $(INSTALL_PATH)/milvus $(PWD)/cmd/main.go 1>/dev/null
 
 get-build-deps:
 	@(env bash $(PWD)/scripts/install_deps.sh)
@@ -122,6 +122,7 @@ integration-test:
     		${APPLE_SILICON_FLAG} -o $(INSTALL_PATH)/integration-test $(INTERATION_PATH)/ 1>/dev/null
 
 BUILD_TAGS = $(shell git describe --tags --always --dirty="-dev")
+BUILD_TAGS_GPU = ${BUILD_TAGS}-gpu
 BUILD_TIME = $(shell date -u)
 GIT_COMMIT = $(shell git rev-parse --short HEAD)
 GO_VERSION = $(shell go version)
@@ -282,6 +283,16 @@ install: milvus
 	-cp -r -P $(PWD)/internal/core/output/lib64/*.so* $(LIBRARY_PATH) 2>/dev/null
 	@echo "Installation successful."
 
+gpu-install: milvus-gpu
+	@echo "Installing binary to './bin'"
+	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/milvus $(GOPATH)/bin/milvus
+	@mkdir -p $(LIBRARY_PATH)
+	-cp -r -P $(PWD)/internal/core/output/lib/*.dylib* $(LIBRARY_PATH) 2>/dev/null
+	-cp -r -P $(PWD)/internal/core/output/lib/*.so* $(LIBRARY_PATH) 2>/dev/null
+	-cp -r -P $(PWD)/internal/core/output/lib64/*.so* $(LIBRARY_PATH) 2>/dev/null
+	@echo "Installation successful."
+
+
 clean:
 	@echo "Cleaning up all the generated files"
 	@find . -name '*.test' | xargs rm -fv
@@ -348,3 +359,4 @@ generate-mockery: getdeps
 	$(PWD)/bin/mockery --name=QueryCoordComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/types --filename=mock_querycoord.go --with-expecter --structname=MockQueryCoord --outpkg=types --inpackage
 	$(PWD)/bin/mockery --name=QueryNodeComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/types --filename=mock_querynode.go --with-expecter --structname=MockQueryNode --outpkg=types --inpackage
 ci-ut: build-cpp-with-coverage generated-proto-go-without-cpp codecov-cpp codecov-go
+

build/build_image_gpu.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# Licensed to the LF AI & Data foundation under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Exit immediately for non zero status
+set -e
+# Print commands
+set -x
+
+# Absolute path to the toplevel milvus directory.
+toplevel=$(dirname "$(cd "$(dirname "${0}")"; pwd)")
+
+OS_NAME="${OS_NAME:-ubuntu20.04}"
+MILVUS_IMAGE_REPO="${MILVUS_IMAGE_REPO:-milvusdb/milvus}"
+MILVUS_IMAGE_TAG="${MILVUS_IMAGE_TAG:-gpu-latest}"
+
+MILVUS_BASE_IMAGE_REPO="${MILVUS_BASE_IMAGE_REPO:-milvusdb/milvus-base}"
+MILVUS_BASE_IMAGE_TAG="gpu-local"
+BUILD_ARGS=""
+
+pushd "${toplevel}"
+BUILD_BASE_IMAGE=${BUILD_BASE_IMAGE:-"false"}
+
+# Seperate base dockerfile to ignore install dependencies when build milvus image
+if [[ ${OS_NAME} == "ubuntu20.04" && ${BUILD_BASE_IMAGE} == "true" ]]; then 
+  docker build -f "./build/docker/milvus/gpu/${OS_NAME}/Dockerfile.base" -t "${MILVUS_BASE_IMAGE_REPO}:${MILVUS_BASE_IMAGE_TAG}" .
+  BUILD_ARGS="--build-arg MILVUS_BASE_IMAGE_REPO=${MILVUS_BASE_IMAGE_REPO} --build-arg MILVUS_BASE_IMAGE_TAG=${MILVUS_BASE_IMAGE_TAG}"
+fi 
+
+docker build ${BUILD_ARGS} -f "./build/docker/milvus/gpu/${OS_NAME}/Dockerfile" -t "${MILVUS_IMAGE_REPO}:${MILVUS_IMAGE_TAG}" .
+
+image_size=$(docker inspect ${MILVUS_IMAGE_REPO}:${MILVUS_IMAGE_TAG}  -f '{{.Size}}'| awk '{ byte =$1 /1024/1024/1024; print byte " GB" }')
+
+echo "Image Size for  ${MILVUS_IMAGE_REPO}:${MILVUS_IMAGE_TAG} is ${image_size}"
+
+popd

build/builder_gpu.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Absolute path to the toplevel milvus directory.
+toplevel=$(dirname "$(cd "$(dirname "${0}")"; pwd)")
+
+export OS_NAME="${OS_NAME:-ubuntu20.04}"
+
+pushd "${toplevel}"
+
+if [[ "${1-}" == "pull" ]]; then
+    docker-compose pull --ignore-pull-failures gpubuilder
+    exit 0
+fi
+
+if [[ "${1-}" == "down" ]]; then
+    docker-compose down
+    exit 0
+fi
+
+# Attempt to run in the container with the same UID/GID as we have on the host,
+# as this results in the correct permissions on files created in the shared
+# volumes. This isn't always possible, however, as IDs less than 100 are
+# reserved by Debian, and IDs in the low 100s are dynamically assigned to
+# various system users and groups. To be safe, if we see a UID/GID less than
+# 500, promote it to 501. This is notably necessary on macOS Lion and later,
+# where administrator accounts are created with a GID of 20. This solution is
+# not foolproof, but it works well in practice.
+uid=$(id -u)
+gid=$(id -g)
+[ "$uid" -lt 500 ] && uid=501
+[ "$gid" -lt 500 ] && gid=$uid
+
+mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/amd64-${OS_NAME}-ccache"
+mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/amd64-${OS_NAME}-go-mod"
+mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/amd64-${OS_NAME}-vscode-extensions"
+mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/amd64-${OS_NAME}-conan"
+chmod -R 777 "${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}"
+
+docker-compose pull --ignore-pull-failures gpubuilder
+if [[ "${CHECK_BUILDER:-}" == "1" ]]; then
+    docker-compose build gpubuilder 
+fi
+
+if [[ "$(id -u)" != "0" ]]; then
+    docker-compose run --no-deps --rm -u "$uid:$gid" gpubuilder "$@"
+else
+    docker-compose run --no-deps --rm --entrypoint "/tini -- /entrypoint.sh" gpubuilder "$@"
+fi
+
+popd

build/docker/builder/gpu/ubuntu20.04/Dockerfile

@@ -0,0 +1,76 @@
+# Copyright (C) 2019-2022 Zilliz. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under the License.
+
+FROM nvidia/cuda:11.6.0-devel-ubuntu20.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-certificates gnupg2 && \
+    wget -qO- "https://cmake.org/files/v3.24/cmake-3.24.0-linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \
+    apt-get update && apt-get install -y --no-install-recommends \
+    g++ gcc gfortran git make ccache libssl-dev zlib1g-dev unzip \
+    clang-format-10 clang-tidy-10 lcov libtool m4 autoconf automake python3 python3-pip \
+    pkg-config uuid-dev libaio-dev libgoogle-perftools-dev libopenblas-dev && \
+    apt-get remove --purge -y && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install conan==1.58.0
+
+# Instal openblas
+# RUN wget https://github.com/xianyi/OpenBLAS/archive/v0.3.21.tar.gz && \
+#    tar zxvf v0.3.21.tar.gz && cd OpenBLAS-0.3.21 && \
+#    make NO_STATIC=1 NO_LAPACK=1 NO_LAPACKE=1 NO_CBLAS=1 NO_AFFINITY=1 USE_OPENMP=1 \
+#       TARGET=HASWELL DYNAMIC_ARCH=1 \
+#       NUM_THREADS=64 MAJOR_VERSION=3 libs shared && \
+#    make PREFIX=/usr/local NUM_THREADS=64 MAJOR_VERSION=3 install && \
+#    rm -f /usr/local/include/cblas.h /usr/local/include/lapack* && \
+#    cd .. && rm -rf OpenBLAS-0.3.21 && rm v0.3.21.tar.gz
+
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib"
+
+# Install Go
+ENV GOPATH /go
+ENV GOROOT /usr/local/go
+ENV GO111MODULE on
+ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
+RUN mkdir -p /usr/local/go && wget -qO- "https://golang.org/dl/go1.18.3.linux-amd64.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \
+    mkdir -p "$GOPATH/src" "$GOPATH/bin" && \
+    curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOPATH}/bin v1.46.2 && \
+    # export GO111MODULE=on && go get github.com/quasilyte/go-ruleguard/cmd/ruleguard@v0.2.1 && \
+    go install github.com/ramya-rao-a/go-outline@latest && \
+    go install golang.org/x/tools/gopls@latest && \
+    go install github.com/uudashr/gopkgs/v2/cmd/gopkgs@latest && \
+    go install github.com/go-delve/delve/cmd/dlv@latest && \
+    go install honnef.co/go/tools/cmd/staticcheck@2022.1 && \
+    go clean --modcache && \
+    chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR)
+
+RUN ln -s /go/bin/dlv /go/bin/dlv-dap
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gdb gdbserver && \
+    apt-get remove --purge -y && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN echo 'root:root' | chpasswd
+
+# refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild
+RUN mkdir -p /home/milvus/.vscode-server/extensions \
+        /home/milvus/.vscode-server-insiders/extensions \
+    && chmod -R 777 /home/milvus
+
+COPY --chown=0:0 build/docker/builder/entrypoint.sh /
+
+RUN wget -qO- "https://github.com/jeffoverflow/autouseradd/releases/download/1.2.0/autouseradd-1.2.0-amd64.tar.gz" | tar xz -C / --strip-components 1
+
+RUN wget -O /tini https://github.com/krallin/tini/releases/download/v0.19.0/tini && \
+    chmod +x /tini
+
+ENTRYPOINT [ "/tini", "--", "autouseradd", "--user", "milvus", "--", "/entrypoint.sh" ]
+CMD ["tail", "-f", "/dev/null"]

build/docker/milvus/gpu/ubuntu20.04/Dockerfile

@@ -0,0 +1,23 @@
+# Copyright (C) 2019-2022 Zilliz. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under the License.
+ARG MILVUS_BASE_IMAGE_REPO="milvusdb/milvus-base"
+ARG MILVUS_BASE_IMAGE_TAG="gpu-20230317-e62607a"
+FROM ${MILVUS_BASE_IMAGE_REPO}:${MILVUS_BASE_IMAGE_TAG}
+
+COPY ./bin/ /milvus/bin/
+
+COPY ./configs/ /milvus/configs/
+
+COPY ./lib/ /milvus/lib/
+
+ENV PATH=/milvus/bin:$PATH
+ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib
+ENV LD_PRELOAD=/milvus/lib/libjemalloc.so

build/docker/milvus/gpu/ubuntu20.04/Dockerfile.base

@@ -0,0 +1,38 @@
+# Copyright (C) 2019-2022 Zilliz. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under the License.
+
+FROM milvusdb/openblas:ubuntu20.04-20220914-179ea77 AS openblas
+
+#FROM alpine
+#FROM ubuntu:focal-20220426
+#ENV DEBIAN_FRONTEND=noninteractive
+
+FROM nvidia/cuda:11.6.0-runtime-ubuntu20.04
+
+ENV TZ=UTC
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends curl libtbb-dev gfortran netcat iputils-ping ca-certificates liblapack3 libzstd-dev uuid-dev libaio-dev libboost-program-options-dev libboost-filesystem-dev && \
+    apt-get remove --purge -y && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY --from=openblas /usr/local/lib/libopenblasp-r0.3.21.so /usr/lib/
+
+RUN ln -s /usr/lib/libopenblasp-r0.3.21.so /usr/lib/libopenblas.so.3 && \
+    ln -s /usr/lib/libopenblas.so.3 /usr/lib/libopenblas.so.0 && \
+    ln -s /usr/lib/libopenblas.so.0 /usr/lib/libopenblas.so
+
+# Add Tini
+ADD https://github.com/krallin/tini/releases/download/v0.19.0/tini /tini
+RUN chmod +x /tini
+ENTRYPOINT ["/tini", "--"]
+
+WORKDIR /milvus/