Refactored the old base and instance images into a new instance image based on minimal-runtime image

Author: Forinil

Dockerfile

@@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1.6
 # Base docker file having defined environment for build and run of hived instance.
 # Modify CI_IMAGE_TAG here and inside script hive/scripts/ci-helpers/build_ci_base_image.sh and run it. Then push images to registry
 # To be started from cloned haf source directory.
@@ -86,22 +87,24 @@ SHELL ["/bin/bash", "-c"]
 # Get everything from cwd as sources to be built.
 COPY --chown=hived_admin:users . /home/hived_admin/source
 
-RUN \
-  mkdir -p ./build/tests/unit/ \
-  && ./source/${HIVE_SUBDIR}/scripts/build.sh --source-dir="./source/${HIVE_SUBDIR}" --binary-dir="./build" \
+RUN <<-EOF
+  set -e
+
+  INSTALLATION_DIR="/home/hived/bin"
+  sudo --user=hived mkdir -p "${INSTALLATION_DIR}"
+
+  ./source/${HIVE_SUBDIR}/scripts/build.sh --source-dir="./source/${HIVE_SUBDIR}" --binary-dir="./build" \
   --cmake-arg="-DBUILD_HIVE_TESTNET=${BUILD_HIVE_TESTNET}" \
   --cmake-arg="-DENABLE_SMT_SUPPORT=${ENABLE_SMT_SUPPORT}" \
   --cmake-arg="-DHIVE_CONVERTER_BUILD=${HIVE_CONVERTER_BUILD}" \
   --cmake-arg="-DHIVE_LINT=${HIVE_LINT}" \
-  && \
-  cd ./build && \
-  find . -name *.o  -type f -delete && \
-  find . -name *.a  -type f -delete
+  --flat-binary-directory="${INSTALLATION_DIR}" \
+  --clean-after-build
 
-# This prevents crash if testnet is not build
-RUN mkdir -p /home/hived_admin/build/libraries/vendor/rocksdb/tools
+  sudo chown -R hived "${INSTALLATION_DIR}/"*
+EOF
 
-FROM ${CI_REGISTRY_IMAGE}runtime:$CI_IMAGE_TAG AS base
+FROM ${CI_REGISTRY_IMAGE}minimal-runtime:$CI_IMAGE_TAG AS instance
 
 ARG BUILD_TIME
 ARG GIT_COMMIT_SHA
@@ -152,13 +155,7 @@ RUN mkdir -p /home/hived/bin && \
     chown -Rc hived:users /home/hived/
 
 COPY --from=build --chown=hived:users \
-  /home/hived_admin/build/programs/hived/hived \
-  /home/hived_admin/build/programs/cli_wallet/cli_wallet \
-  /home/hived_admin/build/programs/beekeeper/beekeeper \
-  /home/hived_admin/build/programs/util/* \
-  /home/hived_admin/build/programs/blockchain_converter/blockchain_converter* \
-  /home/hived_admin/build/tests/unit/* \
-  /home/hived_admin/build/libraries/vendor/rocksdb/tools/sst_dum*  /home/hived/bin/
+    /home/hived/bin/*  /home/hived/bin/
 
 COPY --from=build --chown=hived:users /home/hived_admin/source/${HIVE_SUBDIR}/doc/example_config.ini /home/hived/datadir/example_config.ini
 
@@ -179,13 +176,6 @@ ENV SHM_DIR=${DATADIR}/blockchain
 
 STOPSIGNAL SIGINT
 
-# JSON rpc service
-EXPOSE ${HTTP_PORT}
-
-ENTRYPOINT [ "/home/hived_admin/docker_entrypoint.sh" ]
-
-FROM ${CI_REGISTRY_IMAGE}${IMAGE_TAG_PREFIX}base:${BUILD_IMAGE_TAG} AS instance
-
 #p2p service
 EXPOSE ${P2P_PORT}
 # websocket service
@@ -195,9 +185,4 @@ EXPOSE ${HTTP_PORT}
 # Port specific to HTTP cli_wallet server
 EXPOSE ${CLI_WALLET_PORT}
 
-# We don't have a separate 'minimal-instance' version of hive.  We could create one, based
-# off `minimal-runtime` instead of `runtime` and probably save a hundred MB or so, but
-# the current `instance` isn't bad.  Our current focus is shrinking the haf image size,
-# and since we use the same scripts for building both hive and haf, we need a
-# 'minimal-instance' target here to match the one in haf.  Don't use this.
-FROM instance AS minimal
+ENTRYPOINT [ "/home/hived_admin/docker_entrypoint.sh" ]

scripts/build.sh

@@ -4,30 +4,41 @@ set -euo pipefail
 
 SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 
-LOG_FILE=build.log
-
+export LOG_FILE=build.log
+# shellcheck source=./common.sh
 source "$SCRIPTPATH/common.sh"
 
 log_exec_params "$@"
 
 #This cript builds all (or selected) targets in the hive repo
 
 print_help () {
-    echo "Usage: $0 [OPTION[=VALUE]]... [target]..."
-    echo
-    echo "Allows to build Hive sources "
-    echo "  --source-dir=DIRECTORY_PATH"
-    echo "                       Specify a directory containing a Hived source tree."
-    echo "  --binary-dir=DIRECTORY_PATH"
-    echo "                       Specify a directory to store a build output (hived binaries)."
-    echo "                       Usually it is a \`build\` subdirectory in the hive source tree."
-    echo "  --cmake-arg=ARG      Specify additional arguments to the CMake tool spawn."
-    echo "  --help               Display this help screen and exit."
-    echo
+cat <<EOF
+Usage: $0 [OPTION[=VALUE]]... [target]...
+
+Allows to build Hive sources 
+  --source-dir=DIRECTORY_PATH
+                           Specify a directory containing a Hived source tree.
+  --binary-dir=DIRECTORY_PATH
+                           Specify a directory to store a build output (hived binaries).
+                           Usually it is a \`build\` subdirectory in the hive source tree.
+  --flat-binary-directory=DIRECTORY_PATH
+                           Directory where all binaries are moved after build while flattening
+                           the directory structure, that is installation directory (optional, not set by default, must exist)
+  --cmake-arg=ARG          Specify additional arguments to the CMake tool spawn.
+  --clean-after-build      Remove compiled files after build
+  --haf-build              Set if build is called from HAF
+  --help                   Display this help screen and exit.
+
+EOF
 }
 
 HIVED_BINARY_DIR="../build"
 HIVED_SOURCE_DIR="."
+HIVED_INSTALLATION_DIR=""
+CLEAN_AFTER_BUILD="false"
+HAF_BUILD=""
+
 CMAKE_ARGS=()
 
 JOBS=$(nproc)
@@ -50,6 +61,15 @@ while [ $# -gt 0 ]; do
     --source-dir=*)
         HIVED_SOURCE_DIR="${1#*=}"
         ;;
+    --flat-binary-directory=*)
+        HIVED_INSTALLATION_DIR="${1#*=}"
+        ;;
+    --clean-after-build)
+        CLEAN_AFTER_BUILD="true"
+        ;;
+    --haf-build)
+        HAF_BUILD="true"
+        ;;
     --help)
         print_help
         exit 0
@@ -67,16 +87,69 @@ while [ $# -gt 0 ]; do
     shift
 done
 
-abs_src_dir=`realpath -e --relative-base="$SCRIPTPATH" "$HIVED_SOURCE_DIR"`
-abs_build_dir=`realpath -m --relative-base="$SCRIPTPATH" "$HIVED_BINARY_DIR"`
+abs_src_dir=$(realpath -e --relative-base="$SCRIPTPATH" "$HIVED_SOURCE_DIR")
+abs_build_dir=$(realpath -m --relative-base="$SCRIPTPATH" "$HIVED_BINARY_DIR")
 
-pwd
+# pwd
 mkdir -vp "$abs_build_dir"
 pushd "$abs_build_dir"
-pwd
-cmake -DCMAKE_BUILD_TYPE=Release -GNinja "${CMAKE_ARGS[@]}" "$abs_src_dir"
+# pwd
+
+echo "Building Hive!"
 
+cmake -DCMAKE_BUILD_TYPE=Release -GNinja "${CMAKE_ARGS[@]}" "$abs_src_dir"
 ninja "$@"
 
+if [[ "$CLEAN_AFTER_BUILD" == "true" ]]; then
+    echo "Cleaning up after build..."
+    # Patterns for find must be quoted so they're not expanded by Bash
+    find . -name "*.o"  -type f -delete
+    find . -name "*.a"  -type f -delete
+fi
+
+echo "Done!"
+
 popd
 
+if [[ -d "$HIVED_INSTALLATION_DIR" ]]; then
+
+    # Move all the binaries to the $HIVED_INSTALLATION_DIR directory
+    sudo mv "$abs_build_dir/${HAF_BUILD:+"hive/"}programs/hived/hived" \
+    "$abs_build_dir/${HAF_BUILD:+"hive/"}programs/cli_wallet/cli_wallet" \
+    "$abs_build_dir/${HAF_BUILD:+"hive/"}programs/util/"* \
+    "$HIVED_INSTALLATION_DIR/"
+
+    sudo rm -rf "$HIVED_INSTALLATION_DIR/CMakeFiles"
+
+    if [[ -f "$abs_build_dir/${HAF_BUILD:+"hive/"}programs/beekeeper/beekeeper/beekeeper" ]]; then
+        sudo mv "$abs_build_dir/${HAF_BUILD:+"hive/"}programs/beekeeper/beekeeper/beekeeper" \
+            "$HIVED_INSTALLATION_DIR/"
+    fi
+
+    if [[ -n "$(shopt -s nullglob; echo "$abs_build_dir/${HAF_BUILD:+"hive/"}programs/blockchain_converter/blockchain_converter"*)" ]]; then
+        sudo mv "$abs_build_dir/${HAF_BUILD:+"hive/"}programs/blockchain_converter/blockchain_converter"* \
+            "$HIVED_INSTALLATION_DIR/"
+    fi
+
+    if [[ -n "$(shopt -s nullglob; echo "$abs_build_dir/${HAF_BUILD:+"hive/"}bin/"*)" ]]; then
+        sudo mv "$abs_build_dir/${HAF_BUILD:+"hive/"}bin/"* \
+            "$HIVED_INSTALLATION_DIR/"
+    fi
+
+    if [[ -n "$(shopt -s nullglob; echo "$abs_build_dir/bin/"*)" ]]; then
+        sudo mv "$abs_build_dir/bin/"* \
+            "$HIVED_INSTALLATION_DIR/"
+    fi
+
+    if [[ -n "$(shopt -s nullglob; echo "$abs_build_dir/tests/unit/"*)" ]]; then
+        sudo mv "$abs_build_dir/tests/unit/"* "$HIVED_INSTALLATION_DIR/"
+    fi
+
+    sudo rm -rf "$HIVED_INSTALLATION_DIR/CMakeFiles" "$HIVED_INSTALLATION_DIR/cmake_install.cmake"
+
+    if [[ -n "$(shopt -s nullglob; echo "$abs_build_dir/${HAF_BUILD:+"hive/"}libraries/vendor/rocksdb/tools/sst_dum"*)" ]]; then
+        sudo mv "$abs_build_dir/${HAF_BUILD:+"hive/"}libraries/vendor/rocksdb/tools/sst_dum"* \
+            "$HIVED_INSTALLATION_DIR/"
+    fi
+
+fi

scripts/ci-helpers/build_and_publish_instance.sh

@@ -115,21 +115,21 @@ echo "Building an instance image in the source directory $SRC_DIR"
 "$SRC_DIR/scripts/ci-helpers/build_instance.sh" "$CI_COMMIT_TAG" "$SRC_DIR" "$CI_REGISTRY_IMAGE" "${BUILD_INSTANCE_ADDITIONAL_ARGS[@]}"
 
 echo "Tagging the image built in the previous step as ${CI_REGISTRY_IMAGE}"
-docker tag "$CI_REGISTRY_IMAGE/minimal:$CI_COMMIT_TAG" "$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG"
+docker tag "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}" "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}"
 
-HIVEIO_IMG="hiveio/${CI_PROJECT_NAME}:$CI_COMMIT_TAG"
+HIVEIO_IMG="hiveio/${CI_PROJECT_NAME}:${CI_COMMIT_TAG}"
 
 echo "Tagging the image built in the previous step as ${HIVEIO_IMG}"
-docker tag "$CI_REGISTRY_IMAGE/minimal:$CI_COMMIT_TAG" "${HIVEIO_IMG}"
+docker tag "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}" "${HIVEIO_IMG}"
 
-HIVEBLOG_IMG="registry-upload.hive.blog/${CI_PROJECT_NAME}:$CI_COMMIT_TAG"
+HIVEBLOG_IMG="registry-upload.hive.blog/${CI_PROJECT_NAME}:${CI_COMMIT_TAG}"
 
 echo "Tagging the image built in the previous step as ${HIVEBLOG_IMG}"
-docker tag "$CI_REGISTRY_IMAGE/minimal:$CI_COMMIT_TAG" "${HIVEBLOG_IMG}"
+docker tag "$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG" "${HIVEBLOG_IMG}"
 
 docker images
 
 echo "Pushing instance images"
-docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG"
+docker push "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}"
 docker push "${HIVEIO_IMG}"
 docker push "${HIVEBLOG_IMG}"

scripts/ci-helpers/build_instance.sh

@@ -115,7 +115,6 @@ if [ -z "$GIT_CURRENT_BRANCH" ]; then
   fi
 fi
 
-
 GIT_LAST_LOG_MESSAGE="$(git log -1 --pretty=%B || true)"
 if [ -z "$GIT_LAST_LOG_MESSAGE" ]; then
   GIT_LAST_LOG_MESSAGE="[unknown]"
@@ -131,49 +130,25 @@ if [ -z "$GIT_LAST_COMMIT_DATE" ]; then
   GIT_LAST_COMMIT_DATE="[unknown]"
 fi
 
-echo -e "\nBuilding base instance image...\n"
-
-docker build --target=base \
-  --build-arg CI_REGISTRY_IMAGE="$REGISTRY/" \
-  --build-arg BUILD_HIVE_TESTNET="$BUILD_HIVE_TESTNET" \
-  --build-arg HIVE_CONVERTER_BUILD="$HIVE_CONVERTER_BUILD" \
-  --build-arg BUILD_IMAGE_TAG="$BUILD_IMAGE_TAG" \
-  --build-arg BUILD_TIME="$BUILD_TIME" \
-  --build-arg GIT_COMMIT_SHA="$GIT_COMMIT_SHA" \
-  --build-arg GIT_CURRENT_BRANCH="$GIT_CURRENT_BRANCH" \
-  --build-arg GIT_LAST_LOG_MESSAGE="$GIT_LAST_LOG_MESSAGE" \
-  --build-arg GIT_LAST_COMMITTER="$GIT_LAST_COMMITTER" \
-  --build-arg GIT_LAST_COMMIT_DATE="$GIT_LAST_COMMIT_DATE" \
-  --build-arg HIVE_SUBDIR="$HIVE_SUBDIR" \
-  --build-arg IMAGE_TAG_PREFIX="${IMAGE_TAG_PREFIX:+$IMAGE_TAG_PREFIX-}" \
-  --tag "${REGISTRY}/${IMAGE_TAG_PREFIX:+$IMAGE_TAG_PREFIX-}base:${BUILD_IMAGE_TAG}" \
-  --file Dockerfile "$SOURCE_DIR"
-
-echo -e "\nDone!\nBuilding instance image...\n"
+echo -e "Building Docker build image...\n"
 
-docker build --target=instance \
+docker build --target=build \
   --build-arg CI_REGISTRY_IMAGE="$REGISTRY/" \
   --build-arg BUILD_HIVE_TESTNET=$BUILD_HIVE_TESTNET \
   --build-arg HIVE_CONVERTER_BUILD=$HIVE_CONVERTER_BUILD \
   --build-arg BUILD_IMAGE_TAG="$BUILD_IMAGE_TAG" \
-  --build-arg BUILD_TIME="$BUILD_TIME" \
-  --build-arg GIT_COMMIT_SHA="$GIT_COMMIT_SHA" \
-  --build-arg GIT_CURRENT_BRANCH="$GIT_CURRENT_BRANCH" \
-  --build-arg GIT_LAST_LOG_MESSAGE="$GIT_LAST_LOG_MESSAGE" \
-  --build-arg GIT_LAST_COMMITTER="$GIT_LAST_COMMITTER" \
-  --build-arg GIT_LAST_COMMIT_DATE="$GIT_LAST_COMMIT_DATE" \
   --build-arg HIVE_SUBDIR="$HIVE_SUBDIR" \
   --build-arg IMAGE_TAG_PREFIX="${IMAGE_TAG_PREFIX:+$IMAGE_TAG_PREFIX-}" \
-  --tag "${REGISTRY}${IMAGE_TAG_PREFIX:+/$IMAGE_TAG_PREFIX}:${BUILD_IMAGE_TAG}" \
+  --tag "${REGISTRY}${IMAGE_TAG_PREFIX:+/$IMAGE_TAG_PREFIX}/build:${BUILD_IMAGE_TAG}" \
   --file Dockerfile "$SOURCE_DIR"
 
-echo -e "\nDone!\nBuilding minimal instance image...\n"
+echo -e "\nDone!\nBuilding Docker instance image...\n"
 
-docker build --target=minimal \
+docker build --target=instance \
   --build-arg CI_REGISTRY_IMAGE="$REGISTRY/" \
+  --build-arg BUILD_IMAGE_TAG="$BUILD_IMAGE_TAG" \
   --build-arg BUILD_HIVE_TESTNET=$BUILD_HIVE_TESTNET \
   --build-arg HIVE_CONVERTER_BUILD=$HIVE_CONVERTER_BUILD \
-  --build-arg BUILD_IMAGE_TAG="$BUILD_IMAGE_TAG" \
   --build-arg BUILD_TIME="$BUILD_TIME" \
   --build-arg GIT_COMMIT_SHA="$GIT_COMMIT_SHA" \
   --build-arg GIT_CURRENT_BRANCH="$GIT_CURRENT_BRANCH" \
@@ -182,7 +157,8 @@ docker build --target=minimal \
   --build-arg GIT_LAST_COMMIT_DATE="$GIT_LAST_COMMIT_DATE" \
   --build-arg HIVE_SUBDIR="$HIVE_SUBDIR" \
   --build-arg IMAGE_TAG_PREFIX="${IMAGE_TAG_PREFIX:+$IMAGE_TAG_PREFIX-}" \
-  --tag "${REGISTRY}/${IMAGE_TAG_PREFIX:+$IMAGE_TAG_PREFIX-}minimal:${BUILD_IMAGE_TAG}" \
+  --build-context "build=docker-image://${REGISTRY}${IMAGE_TAG_PREFIX:+/$IMAGE_TAG_PREFIX}/build:${BUILD_IMAGE_TAG}" \
+  --tag "${REGISTRY}${IMAGE_TAG_PREFIX:+/$IMAGE_TAG_PREFIX}:${BUILD_IMAGE_TAG}" \
   --file Dockerfile "$SOURCE_DIR"
 
 echo -e "\nDone!\n"
@@ -191,6 +167,6 @@ popd
 
 if [ -n "${EXPORT_PATH}" ];
 then
-  "$SCRIPTPATH/export-data-from-docker-image.sh" "${REGISTRY}/${IMAGE_TAG_PREFIX:+$IMAGE_TAG_PREFIX-}base:${BUILD_IMAGE_TAG}" "${EXPORT_PATH}"
+  "$SCRIPTPATH/export-data-from-docker-image.sh" "${REGISTRY}${IMAGE_TAG_PREFIX:+/$IMAGE_TAG_PREFIX}/build:${BUILD_IMAGE_TAG}" "${EXPORT_PATH}"
 fi