stage-build

#!/bin/bash

back_off() {
  local command=$1
  local max_retries=${2:-5}
  local count=0

  while true; do
    $command && return 0
    ((count++))

    [[ $count -eq $max_retries ]] && return 1

    sleep_time=$((2**count))
    echo "Retrying in $sleep_time seconds"
    sleep $sleep_time
  done
}

datetime () { date +"%Y-%m-%d %H:%M:%S"; }

[[ -t 1 ]] && interm=yes

red ()    { echo "\e[1;31m$1\e[m"; }
yellow () { echo "\e[1;33m$1\e[m"; }
log () { printf "$(yellow "[log $(datetime)]") $1\n"; }
err () { printf "$(red "[error $(datetime)]") $1\n"; exit 1; }

# return a unique id for the build
get_build_id () {
    [[ -z "${CI_PIPELINE_ID}" ]] && (cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 10) || echo ${CI_PIPELINE_ID}
}

usage () {
    echo "usage: stage-build -n name -r recipe-path -s system -b build-root -u uarch -m mount [-d] [-w] [-t]"
    echo ""
    echo "where:"
    echo "  name:        the name of the stack"
    echo "  system:      the cluster name \(e.g. one of balfrin, todi, eiger \)"
    echo "  recipe-path: the location of the recipe"
    echo '  build-root:  the root path in which to perform the build, e.g. /dev/shm/$USER'
    echo "  uarch:       the target micro-architecture"
    echo "  mount:       the mount point of the image"
    echo ""
    echo "-w: enable no-bwrap mode"
    echo "-t: disable pushing artifacts to registry"
    [[ "" == "$1" ]] && exit 0
    err "$1"
}

name="-"
system="-"
recipe_path="-"
build_root="-"
uarch="-"
mount="-"
no_bwrap=""
# if this is set to "yes", assume we are doing a manual test run and do
# not push to jfrog
no_push=""

# print numactl information
numactl -s

log "repo: $(git log -1 --pretty=format:'%D %h %s')"

echo "====== parsing arguments $@"

while getopts n:s:r:b:u:m:dwt flag
do
    case "${flag}" in
        n) name=${OPTARG};;
        s) system=${OPTARG};;
        r) recipe_path=$(realpath "${OPTARG}");;
        b) build_root="${OPTARG}";;
        u) uarch=${OPTARG};;
        m) mount=${OPTARG};;
        w) no_bwrap="--no-bwrap";;
        t) no_push="yes";;
    esac
done

[[ "-" == "${name}"        ]] && usage "missing name argument"
[[ "-" == "${system}"      ]] && usage "missing system argument"
[[ "-" == "${recipe_path}" ]] && usage "missing recipe-path argument"
[[ "-" == "${build_root}"  ]] && usage "missing build-root argument"
[[ "-" == "${uarch}"       ]] && usage "missing uarch argument"
[[ "-" == "${mount}"       ]] && usage "missing mount argument"

build_id=$(get_build_id)
build_path="${build_root}/${name}/${build_id}"
ci_path="$(pwd)"
pipeline_path=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)

log "SCRATCH        ${SCRATCH}"
log "USER           ${USER}"
log "name           ${name}"
log "system         ${system}"
log "recipe-path    ${recipe_path}"
log "build-id       ${build_id}"
log "build-root     ${build_root}"
log "build-path     ${build_path}"
log "ci-path        ${ci_path}"
log "pipeline-path  ${pipeline_path}"
log "mount          ${mount}"
[[ ! -z ${no_bwrap} ]] && log "no bwrap"
[[ "${no_push}" == "yes" ]] && log "disabled push to registry"

# Check if the recipe path exists
[[ -d "${recipe_path}" ]] || err "recipe path '$recipe_path' does not exist"

# add pipeline directory to PATH
export PATH="${pipeline_path}:$PATH"

# build a list of environment variables with the UENV_CIBUILDV_ prefix
# in order to forward them to the make call that builds the uenv, and
# set them in this environment
BUILD_VARS=()
BUILD_VAR_NAMES=()
for civarname in ${!UENV_CIBUILDV_*} ;
do
    # NOTE: take care to print only the variable name, and not its value, to not leak secrets
    log "setting uenv build variable '${civarname#UENV_CIBUILDV_}'"
    BUILD_VARS+=("${civarname#UENV_CIBUILDV_}=${!civarname}")
    BUILD_VAR_NAMES+=("${civarname#UENV_CIBUILDV_}")
done
for var in "${BUILD_VARS[@]}"; do
    export "$var"
done
# append the other variables that are required in the make environment
BUILD_VARS+=("PATH=/usr/bin:/bin:${build_path}/spack/bin")
BUILD_VARS+=("HOME=$HOME")
BUILD_VAR_NAMES+=("PATH" "HOME")

##
# Set up build cache flags if a build cache is available
##

mirror_path=$SCRATCH/uenv-cache
if [ -d "${mirror_path}" ]; then
    log "found build cache at $mirror_path"
    echo "root: $mirror_path" > cache.yaml
    log "cache file: $(realpath cache.yaml) in working path $(pwd)"
    key_path="${mirror_path}/push-key.gpg"
    if [[ -f "${key_path}" ]]; then
        log "found build cache key at $key_path"
        echo "key: ${mirror_path}/push-key.gpg" >> cache.yaml
    fi
    log "cache.yaml:"
    cat cache.yaml
    cache_flags="--cache $(realpath cache.yaml)"
else
    log "no build cache found at $mirror_path"
    rm -f cache.yaml
    cache_flags=""
fi

##
# Download dependencies
##

source ${pipeline_path}/util/setup-dependencies

if [ "$no_push" != "yes" ]; then
    source ${pipeline_path}/util/setup-credentials
fi

##
# Clone the stackinator and alps cluster configuration
##

# Initialise the stackinator
source ${pipeline_path}/util/setup-stackinator

##
# Configure the build
##

# call stackinator
log "configuring stack"

echo "stack-config -r ${recipe_path} -b ${build_path} -s $cluster_path/${system} -m ${mount} $cache_flags ${no_bwrap}"
stack-config -r "${recipe_path}" -b "${build_path}" -s "$cluster_path/${system}"  -m "${mount}" $cache_flags ${no_bwrap}

if [ ! $? -eq 0 ]; then
    # delete stackinator tool once it is no longer needed
    rm -rf "$tool_path" "$cluster_path"
    err "stack-config failed"
fi

# delete stackinator tool once it is no longer needed
rm -rf "$tool_path" "$cluster_path"

##
# Build the Spack stack
##

cd "${build_path}"

log "building image in ${build_path}"

### WARNING: only print variable names BUILD_VAR_NAMES, not the values
log "note, environment variable values are not printed"
echo "env --ignore-environment ${BUILD_VAR_NAMES[@]} make store.squashfs -j64"
env --ignore-environment "${BUILD_VARS[@]}" make store.squashfs -kj64
build_retcode=$?
#
# if the build failed, try again and back off to only 8 tasks
[[ ${build_retcode} -eq 0  ]] || env --ignore-environment "${BUILD_VARS[@]}" make store.squashfs -kj8
build_retcode=$?
# if build failed, push built packages to the cache
[[ ${build_retcode} -eq 0 ]] || env --ignore-environment "${BUILD_VARS[@]}" make cache-force

if [ ! ${build_retcode} -eq 0 ]; then
    log "$(ls ${build_path}/tmp)"

    logpath="$CI_PROJECT_DIR/build-logs"
    log "saving logs to $logpath"
    rm -rf "$logpath"
    mkdir "$logpath"
    cp "${build_path}/spack-bootstrap-output" "$logpath"
    cp -r "${build_path}/tmp" $logpath
    err "error building image"
fi

if [ "$no_push" == "yes" ]; then
    exit 0;
fi

##
# Push build artifacts to JFrog
##

rego=jfrog.svc.cscs.ch/uenv
repo_base=build/${system}/${uarch}/${name}
repo=${repo_base}:${build_id}
outpath_repo=${rego}/${repo}
if [[ -n "$REPOSITORY_OVERRIDE_NAME" ]] ; then
    # Allow overriding path, where the data is stored.
    # This can be a remote path, or a local path
    outpath_repo="${REPOSITORY_OVERRIDE_NAME}"
fi

if [[ "${outpath_repo}" == "/"* ]] ; then
    # local path
    log "Copying store.squashfs and meta to ${outpath_repo}"
    mkdir -p "$(dirname "${outpath_repo}")"
    cp store.squashfs "${outpath_repo}"
    cp -r store/meta "${outpath_repo}.meta"
else
    log "pushing store.squashfs to ${outpath_repo}"
    log "oras push --registry-config ${jfrog_creds_path} ${outpath_repo} --artifact-type application/x-squashfs store.squashfs"
    oras push --registry-config ${jfrog_creds_path} ${outpath_repo} --artifact-type application/x-squashfs store.squashfs
    [[ $? -eq 0  ]] || err "failed to push image"

    # push the metadata to jfrog
    log "pushing meta data to ${outpath_repo}"
    log "oras attach --registry-config ${jfrog_creds_path} --artifact-type uenv/meta ${outpath_repo} ./meta"
    (cd ./store; oras attach --registry-config ${jfrog_creds_path} --artifact-type uenv/meta ${outpath_repo} ./meta)
fi

log "clean up build path '${build_path}'"
(rm -rf ${build_path})