etcd-common-tools

# Common environment variables
ASSET_DIR="/home/core/assets"
CONFIG_FILE_DIR="/etc/kubernetes"
MANIFEST_DIR="${CONFIG_FILE_DIR}/manifests"
ETCD_DATA_DIR="/var/lib/etcd"
ETCD_DATA_DIR_BACKUP="/var/lib/etcd-backup"
ETCD_REV_JSON="${ETCD_DATA_DIR}/revision.json"
MANIFEST_STOPPED_DIR="${ASSET_DIR}/manifests-stopped"
RESTORE_ETCD_POD_YAML="${CONFIG_FILE_DIR}/static-pod-resources/etcd-certs/configmaps/restore-etcd-pod/pod.yaml"
QUORUM_RESTORE_ETCD_POD_YAML="${CONFIG_FILE_DIR}/static-pod-resources/etcd-certs/configmaps/restore-etcd-pod/quorum-restore-pod.yaml"
ETCDCTL_BIN_DIR="${CONFIG_FILE_DIR}/static-pod-resources/bin"
PATH=${PATH}:${ETCDCTL_BIN_DIR}
export KUBECONFIG="/etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/node-kubeconfigs/localhost.kubeconfig"
export ETCD_ETCDCTL_BIN="etcdctl"

# download etcdctl from download release image
function dl_etcdctl {
  # Avoid caching the binary when podman exists, the etcd image is always available locally and we need a way to update etcdctl.
  # When we're running from an etcd image there's no podman and we can continue without a download.
  if ([ -n "$(command -v podman)" ]); then
     local etcdimg=${ETCD_IMAGE}
     local etcdctr=$(podman create --authfile=/var/lib/kubelet/config.json ${etcdimg})
     local etcdmnt=$(podman mount "${etcdctr}")
     [ ! -d ${ETCDCTL_BIN_DIR} ] && mkdir -p ${ETCDCTL_BIN_DIR}
     cp ${etcdmnt}/bin/etcdctl ${ETCDCTL_BIN_DIR}/
     if [ -f "${etcdmnt}/bin/etcdutl" ]; then
       cp ${etcdmnt}/bin/etcdutl ${ETCDCTL_BIN_DIR}/
       export ETCD_ETCDUTL_BIN=etcdutl
     fi
     if ! [ -x "$(command -v jq)" ]; then
       cp ${etcdmnt}/bin/jq ${ETCDCTL_BIN_DIR}/
     fi

     umount "${etcdmnt}"
     podman rm "${etcdctr}"
     etcdctl version
     return
  fi

  if ([ -x "$(command -v etcdctl)" ]); then
    echo "etcdctl is already installed"
    if [ -x "$(command -v etcdutl)" ]; then
      echo "etcdutl is already installed"
      export ETCD_ETCDUTL_BIN=etcdutl
    fi

    return
  fi

  echo "Could neither pull etcdctl nor find it locally in cache. Aborting!"
  exit 1
}

function check_snapshot_status() {
  local snap_file="$1"

  ETCD_CLIENT="${ETCD_ETCDCTL_BIN}"
  if [ -n "${ETCD_ETCDUTL_BIN}" ]; then
    ETCD_CLIENT="${ETCD_ETCDUTL_BIN}"
  fi

  if ! ${ETCD_CLIENT} snapshot status "${snap_file}" -w json; then
    echo "Backup integrity verification failed. Backup appears corrupted. Aborting!"
    return 1
  fi
}

function wait_for_containers_to_stop() {
  local containers=("$@")

  for container_name in "${containers[@]}"; do
    echo "Waiting for container ${container_name} to stop"
    while [[ -n $(crictl ps --label io.kubernetes.container.name="${container_name}" -q) ]]; do
      echo -n "."
      sleep 1
    done
    echo "complete"
  done
}

function backup_remaining_etcd_data_dir_contents() {
  local entry base extras_dir="${ETCD_DATA_DIR_BACKUP}/extra-data-dir-contents"

  mkdir -p "${extras_dir}"

  shopt -s nullglob dotglob
  for entry in "${ETCD_DATA_DIR}"/*; do
    base=$(basename "${entry}")
    if [ -e "${extras_dir}/${base}" ]; then
      echo "removing previous backup ${extras_dir}/${base}"
      rm -rf "${extras_dir:?}/${base}"
    fi
    echo "Moving ${entry} to ${extras_dir}/"
    mv "${entry}" "${extras_dir}/"
  done
  shopt -u nullglob dotglob
}

function mv_static_pods() {
  local containers=("$@")

  # Move manifests and stop static pods
  if [ ! -d "$MANIFEST_STOPPED_DIR" ]; then
    mkdir -p "$MANIFEST_STOPPED_DIR"
  fi

  for POD_FILE_NAME in "${containers[@]}"; do
    echo "...stopping ${POD_FILE_NAME}"
    [ ! -f "${MANIFEST_DIR}/${POD_FILE_NAME}" ] && continue
    mv "${MANIFEST_DIR}/${POD_FILE_NAME}" "${MANIFEST_STOPPED_DIR}"
  done
}

function wait_for_podman_etcd_to_stop() {
  echo "waiting for podman-etcd to stop..."
  local start=$SECONDS
  local timeout=120

  while [ $((SECONDS - start)) -lt "$timeout" ]; do
    local location
    if ! location=$(crm_resource --resource etcd --locate); then
      echo "could not detect if etcd is running. Retrying..."
    elif [ -z "$location" ]; then
      echo "etcd is stopped"
      return 0
    fi

    sleep 1
  done

  echo "timed out waiting for etcd to stop"
  return 1
}

function print_restore_completion_message() {
  cat << 'EOF'
==============================================================================
SNAPSHOT RESTORE COMPLETED
==============================================================================

NEXT STEPS: Monitor the rollout from a system with 'oc' access:

1. Check if the etcd operator completes the rollout successfully:
    $ oc adm wait-for-stable-cluster  # OR oc get co/etcd -w
    Note: It may take a few moments for the rollout to initiate before progress is reflected.

2. IF the rollout does NOT complete, apply the forceRedeploymentReason patch:
    $ oc patch etcd cluster -p="{\"spec\":{\"forceRedeploymentReason\":\"tnf-restore-$(date +%s)\"}}" --type=merge

    Then monitor the rollout again.

3. IF /var/lib/etcd/revision.json is still not being re-created on this node after the patch, manually restart the static pods:

    On the current node:
    $ ssh core@<node_ip> 'sudo mv /etc/kubernetes/manifests/etcd-pod.yaml /tmp/ && sleep 30 && sudo mv /tmp/etcd-pod.yaml /etc/kubernetes/manifests/'

==============================================================================
EOF
}