Allow for scripts in CKS ISO to fully control CKS deployment

plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml

@@ -89,6 +89,22 @@ write_files:
         export PATH=$PATH:/opt/bin
       fi
 
+      if [ -f "${BINARIES_DIR}/scripts/control-node-add/setup-kube-system" ]; then
+        # This setup script will copy any necessary files out of the iso, and possibly overwrite /opt/bin/deploy-kube-system for the second stage
+        if /bin/bash ${BINARIES_DIR}/scripts/control-node-add/setup-kube-system ${BINARIES_DIR} "{{ k8s_control_node.join_ip }}" "{{ k8s_control_node.cluster.token }}" "{{ k8s_control_node.cluster.ha.certificate.key }}" "{{registry.url}}" "{{registry.url.endpoint}}" "{{registry.username}}" "{{registry.password}}" "{{registry.token}}"; then
+          rc=0
+          echo "Success running setup-kube-system on a control plane node add from ISO."
+        else
+          rc=$?
+          echo "Failed running setup-kube-system on a control plane node add from ISO."
+        fi
+        umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
+        if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
+          eject "${iso_drive_path}"
+        fi
+        exit $rc
+      fi
+
       if [ -d "$BINARIES_DIR" ]; then
         ### Binaries available offline ###
         echo "Installing binaries from ${BINARIES_DIR}"

plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml

@@ -109,6 +109,22 @@ write_files:
         export PATH=$PATH:/opt/bin
       fi
 
+      if [ -f "${BINARIES_DIR}/scripts/control-node/setup-kube-system" ]; then
+        # This setup script will copy any necessary files out of the iso, and possibly overwrite /opt/bin/deploy-kube-system, /opt/bin/setup-containerd, etc
+        if /bin/bash ${BINARIES_DIR}/scripts/control-node/setup-kube-system ${BINARIES_DIR} "{{ k8s_control_node.cluster.token }}" "{{ k8s_control_node.cluster.initargs }}" "{{registry.url}}" "{{registry.url.endpoint}}" "{{registry.username}}" "{{registry.password}}" "{{registry.token}}"; then
+          rc=0
+          echo "Success running setup-kube-system on control plane node from ISO."
+        else
+          rc=$?
+          echo "Failed running setup-kube-system on control plane node from ISO."
+        fi
+        umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
+        if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
+          eject "${iso_drive_path}"
+        fi
+        exit $rc
+      fi
+
       if [ -d "$BINARIES_DIR" ]; then
         ### Binaries available offline ###
         echo "Installing binaries from ${BINARIES_DIR}"

plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml

@@ -89,6 +89,22 @@ write_files:
         export PATH=$PATH:/opt/bin
       fi
 
+      if [ -f "${BINARIES_DIR}/scripts/node/setup-kube-system" ]; then
+        # This setup script will copy any necessary files out of the iso, and possibly overwrite /opt/bin/deploy-kube-system, /opt/bin/setup-containerd, etc
+        if /bin/bash ${BINARIES_DIR}/scripts/node/setup-kube-system ${BINARIES_DIR} "{{ k8s_control_node.join_ip }}" "{{ k8s_control_node.cluster.token }}" "{{registry.url}}" "{{registry.url.endpoint}}" "{{registry.username}}" "{{registry.password}}" "{{registry.token}}"; then
+          rc=0
+          echo "Success running setup-kube-system on worker node from ISO."
+        else
+          rc=$?
+          echo "Failed running setup-kube-system on worker node from ISO."
+        fi
+        umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
+        if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
+          eject "${iso_drive_path}"
+        fi
+        exit $rc
+      fi
+
       if [ -d "$BINARIES_DIR" ]; then
         ### Binaries available offline ###
         echo "Installing binaries from ${BINARIES_DIR}"

scripts/util/cks-samples/README.md

@@ -0,0 +1,24 @@
+# CloudStack Kubernetes Service Cloud-init Overrides - Examples
+
+Example files that can be used when creating the CloudStack Kubernetes Service ISO file to override the normal behavior of the Kubernetes cluster creation
+
+
+## Usage
+
+### The basic directory
+
+In the basic directory, there is a scripts directory that can be used as a template to start the customization. It is basically the original cloud-init scripts, but they can then be modified as needed for the CKS cluster. To start, just copy the scripts directory into the util directory, peer with the create-kubernetes-binaries-iso.sh script.
+
+### The cilium directory
+
+In the cilium directory, there is a modified version of create-kubernetes-binaries-iso.sh which contains what would be needed to install Helm and install Cilium instead of Weave. Helm needs to be installed and in the path on the OS where the ISO build will be run. There is no need to customize anything under the cilium directory if you are only looking to run Cilium instead of Weave, but just like the basic directory, other aspects can be customized.
+
+
+## References in CloudStack
+
+The following cloud-init scripts will call the setup-kube-system scripts found here when the cluster is created:
+
+* plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml
+* plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml
+* plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml
+

scripts/util/cks-samples/basic/scripts/control-node-add/deploy-kube-system.tmpl

@@ -0,0 +1,27 @@
+#!/bin/bash -e
+
+if [[ -f "/home/cloud/success" ]]; then
+  echo "Already provisioned!"
+  exit 0
+fi
+
+if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then
+  echo "setup-kube-system is running!"
+  exit 1
+fi
+modprobe ip_vs
+modprobe ip_vs_wrr
+modprobe ip_vs_sh
+modprobe nf_conntrack
+if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
+  export PATH=$PATH:/opt/bin
+fi
+kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --control-plane --certificate-key {{ k8s_control_node.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification
+
+mkdir -p /root/.kube
+cp -i /etc/kubernetes/admin.conf /root/.kube/config
+chown $(id -u):$(id -g) /root/.kube/config
+
+sudo touch /home/cloud/success
+echo "true" > /home/cloud/success
+

scripts/util/cks-samples/basic/scripts/control-node-add/setup-containerd.tmpl

@@ -0,0 +1,13 @@
+#!/bin/bash -e
+
+export registryConfig="\\        [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\         endpoint = [\"{{registry.url}}\"]"
+export registryCredentials="\\      [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\""
+
+echo "creating config file for containerd"
+containerd config default > /etc/containerd/config.toml
+sed  -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml
+sed  -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml
+
+echo "Restarting containerd service"
+systemctl daemon-reload
+systemctl restart containerd

scripts/util/cks-samples/basic/scripts/control-node-add/setup-kube-system

@@ -0,0 +1,138 @@
+#!/bin/bash -e
+
+if [ $# -lt 9 ]; then
+  echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN CERTIFICATE_KEY REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2
+  exit 1
+fi
+
+BINARIES_DIR="$1"
+CLUSTER_JOIN_IP="$2"
+CLUSTER_TOKEN="$3"
+CERTIFICATE_KEY="$4"
+REGISTRY_URL="$5"
+REGISTRY_URL_ENDPOINT="$6"
+REGISTRY_USERNAME="$7"
+REGISTRY_PASSWORD="$8"
+REGISTRY_TOKEN="$9"
+
+ATTEMPT_ONLINE_INSTALL=false
+setup_complete=false
+
+MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
+crucial_cmd_attempts=1
+
+if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
+  export PATH=$PATH:/opt/bin
+fi
+
+if [ -f ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl ]; then
+  sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.ha.certificate.key }}/${CERTIFICATE_KEY}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system
+fi
+
+if [ -f ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl ]; then
+  sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl > /opt/bin/setup-containerd
+fi
+
+if [ -d "$BINARIES_DIR" ]; then
+  ### Binaries available offline ###
+  echo "Installing binaries from ${BINARIES_DIR}"
+  mkdir -p /opt/cni/bin
+  tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz
+
+  mkdir -p /opt/bin
+  tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz
+
+  mkdir -p /opt/bin
+  cd /opt/bin
+  cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} .
+  chmod +x {kubeadm,kubelet,kubectl}
+
+  sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service
+  mkdir -p /etc/systemd/system/kubelet.service.d
+  sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+
+  echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet
+
+  output=`ls ${BINARIES_DIR}/docker/`
+  if [ "$output" != "" ]; then
+    while read -r line; do
+      crucial_cmd_attempts=1
+      while true; do
+        if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then
+          echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!"
+          break;
+        fi
+        retval=0
+        set +e
+        ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line"
+        retval=$?
+        set -e
+        if [ $retval -eq 0 ]; then
+          break;
+        fi
+        crucial_cmd_attempts=$[$crucial_cmd_attempts + 1]
+      done
+    done <<< "$output"
+    setup_complete=true
+  fi
+  if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then
+    mkdir -p /opt/autoscaler
+    cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml
+  fi
+  if [ -e "${BINARIES_DIR}/provider.yaml" ]; then
+    mkdir -p /opt/provider
+    cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml
+  fi
+
+  PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1`
+  echo $PAUSE_IMAGE
+  if [ -n "$PAUSE_IMAGE" ]; then
+    sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml
+  fi
+  systemctl daemon-reload
+  systemctl restart containerd
+
+fi
+if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then
+  ###  Binaries not available offline ###
+  RELEASE="v1.16.3"
+  CNI_VERSION="v0.7.5"
+  CRICTL_VERSION="v1.16.0"
+  echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet."
+  mkdir -p /opt/cni/bin
+  curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz
+
+  mkdir -p /opt/bin
+  curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz
+
+  mkdir -p /opt/bin
+  cd /opt/bin
+  curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl}
+  chmod +x {kubeadm,kubelet,kubectl}
+
+  curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service
+  mkdir -p /etc/systemd/system/kubelet.service.d
+  curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+fi
+
+systemctl enable kubelet && systemctl start kubelet
+modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1
+
+if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then
+  crucial_cmd_attempts=1
+  while true; do
+    if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then
+      echo "Warning: kubeadm pull images failed after multiple tries!"
+      break;
+    fi
+    retval=0
+    set +e
+    kubeadm config images pull --cri-socket /run/containerd/containerd.sock
+    retval=$?
+    set -e
+    if [ $retval -eq 0 ]; then
+      break;
+    fi
+    crucial_cmd_attempts=$[$crucial_cmd_attempts + 1]
+  done
+fi

scripts/util/cks-samples/basic/scripts/control-node/deploy-kube-system.tmpl

@@ -0,0 +1,62 @@
+#!/bin/bash -e
+
+if [[ -f "/home/cloud/success" ]]; then
+echo "Already provisioned!"
+exit 0
+fi
+
+if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
+  export PATH=$PATH:/opt/bin
+fi
+
+MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
+crucial_cmd_attempts=1
+while true; do
+  if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then
+    echo "Error: kubeadm init failed!"
+    exit 1
+  fi
+  retval=0
+  set +e
+  kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock
+  retval=$?
+  set -e
+  if [ $retval -eq 0 ]; then
+    break;
+  fi
+  crucial_cmd_attempts=$[$crucial_cmd_attempts + 1]
+done
+
+K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/
+
+if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then
+  echo "setup-kube-system is running!"
+  exit 1
+fi
+if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
+  export PATH=$PATH:/opt/bin
+fi
+export KUBECONFIG=/etc/kubernetes/admin.conf
+
+mkdir -p /root/.kube
+cp -i /etc/kubernetes/admin.conf /root/.kube/config
+chown $(id -u):$(id -g) /root/.kube/config
+echo export PATH=\$PATH:/opt/bin >> /root/.bashrc
+
+if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then
+  ### Network, dashboard configs available offline ###
+  echo "Offline configs are available!"
+  /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/network.yaml
+  /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/dashboard.yaml
+  rm -rf "${K8S_CONFIG_SCRIPTS_COPY_DIR}"
+else
+  /opt/bin/kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(/opt/bin/kubectl version | base64 | tr -d '\n')"
+  /opt/bin/kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta6/aio/deploy/recommended.yaml
+fi
+
+/opt/bin/kubectl create rolebinding admin-binding --role=admin --user=admin || true
+/opt/bin/kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true
+/opt/bin/kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true
+
+sudo touch /home/cloud/success
+echo "true" > /home/cloud/success

scripts/util/cks-samples/basic/scripts/control-node/setup-containerd.tmpl

@@ -0,0 +1,13 @@
+#!/bin/bash -e
+
+export registryConfig="\\        [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\         endpoint = [\"{{registry.url}}\"]"
+export registryCredentials="\\      [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\""
+
+echo "creating config file for containerd"
+containerd config default > /etc/containerd/config.toml
+sed  -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml
+sed  -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml
+
+echo "Restarting containerd service"
+systemctl daemon-reload
+systemctl restart containerd