Merge pull request #885 from areed/nlb-2

Make Kubernetes API health checks more reliable

Author: areed

scripts/common/kubernetes.sh

@@ -169,7 +169,7 @@ function kubernetes_api_address() {
 }
 
 function kubernetes_api_is_healthy() {
-    curl --noproxy "*" --fail --silent --insecure "https://$(kubernetes_api_address)/healthz"
+    curl --noproxy "*" --fail --silent --insecure "https://$(kubernetes_api_address)/healthz" >/dev/null
 }
 
 function spinner_kubernetes_api_healthy() {
@@ -178,6 +178,15 @@ function spinner_kubernetes_api_healthy() {
     fi
 }
 
+# With AWS NLB kubectl commands may fail to connect to the Kubernetes API immediately after a single
+# successful health check
+function spinner_kubernetes_api_stable() {
+	for i in {1..10}; do
+		sleep 1
+		spinner_kubernetes_api_healthy
+	done
+}
+
 function kubernetes_drain() {
     kubectl drain "$1" \
         --delete-local-data \

scripts/common/upgrade.sh

@@ -66,14 +66,14 @@ function upgrade_kubernetes_local_master_patch() {
 
     kubernetes_drain "$node"
 
-    spinner_kubernetes_api_healthy
+    spinner_kubernetes_api_stable
     kubeadm upgrade apply "v$k8sVersion" --yes --config /opt/replicated/kubeadm.conf --force
     sed -i "s/kubernetesVersion:.*/kubernetesVersion: v${k8sVersion}/" /opt/replicated/kubeadm.conf
 
     kubernetes_install_host_packages "$k8sVersion"
     systemctl daemon-reload
     systemctl restart kubelet
-    spinner_kubernetes_api_healthy
+    spinner_kubernetes_api_stable
     kubectl uncordon "$node"
 
     spinner_until 120 kubernetes_node_has_version "$node" "$k8sVersion"
@@ -178,7 +178,7 @@ EOF
     printf "${YELLOW}Drain local node and apply upgrade? ${NC}"
     confirmY " "
 
-    spinner_kubernetes_api_healthy
+    spinner_kubernetes_api_stable
     kubeadm upgrade apply "v$k8sVersion" --yes --config /opt/replicated/kubeadm.conf --force
     upgrade_etcd_image_18 "$k8sVersion"
     sed -i "s/kubernetesVersion:.*/kubernetesVersion: v${k8sVersion}/" /opt/replicated/kubeadm.conf
@@ -187,7 +187,7 @@ EOF
     systemctl daemon-reload
     systemctl restart kubelet
 
-    spinner_kubernetes_api_healthy
+    spinner_kubernetes_api_stable
     kubectl uncordon "$node"
 
     # force deleting the cache because the api server will use the stale API versions after kubeadm upgrade

scripts/install.sh

@@ -133,6 +133,8 @@ EOF
         spinner_until 120 cert_has_san "$PRIVATE_ADDRESS:6443" "$LOAD_BALANCER_ADDRESS"
     fi
 
+    spinner_kubernetes_api_stable
+
     exportKubeconfig
     KUBEADM_TOKEN_CA_HASH=$(cat /tmp/kubeadm-init | grep 'discovery-token-ca-cert-hash' | awk '{ print $2 }' | head -1)
 
@@ -173,7 +175,6 @@ EOF
         fi
     fi
 
-    spinner_kubernetes_api_healthy
     labelNodes
     kubectl cluster-info