Merge pull request #5 from kairos-io/multi-node-fixes

Added fixes for reconfiguring calico and dqlite for multinode clusters

Author: venkatnsrinivasan

README.md

@@ -25,17 +25,23 @@ This provider will configure a Microk8s installation based on the cluster sectio
           `writeKubeconfig`: "/run/kubeconfig" 
           
           # Switch dqlite to use the internal IP of the Node instead of the 127.0.0.1 
-          `dqliteUseHostIPV4Addres`: true
+          `dqliteUseHostIPV4Address`: true
           
           # Uses the  DNS server entries from the host for the coredns configuration(reads from /etc/resolv.conf)
           `useHostDNS`: true
           
           # Specifies custom DNs server. Overrides the previous setting
           `DNS` : 75.75.74.74
+          # Customize calico settings
+          `calico`: 
+             # Enable IpinIP
+             `calicoIPinIP`: true
+             # Change the calico IP_AUTODETECTION_METHOD env . By default When forming a MicroK8s cluster, Calico is updated to use address that was used in the microk8s join command (IP_AUTODETECTION_METHOD=can-reach=10.10.10.10)
+             `calicoAutoDetect`: "cidr=10.10.128.0/18"  
   -  `initConfiguration`: Configuration only for the init node
-  
-                  addons:
-                    - dns
+
+           addons:
+              - dns
 
 ### Example
 ```yaml
@@ -52,6 +58,9 @@ cluster:
           writeKubeconfig: "/run/kubeconfig"
           dqliteUseHostIPV4Address: true
           useHostDNS: true
+          calico: 
+             calicoIPinIP: true
+             calicoAutoDetect: "cidr=10.10.128.0/18"
     initConfiguration:
                   addons:
                     - dns

main.go

@@ -20,8 +20,10 @@ const (
 	scriptBasePath                  = "/opt/microk8s/scripts/"
 	installMicrok8sScript           = "00-install-microk8s.sh"
 	upgradeMicrok8sScript           = "00-upgrade-microk8s.sh"
-	configureApiServerScript        = "10-configure-apiserver.sh"
+//	configureApiServerScript        = "10-configure-apiserver.sh"
+	configureAltNamesScript         = "10-configure-alt-names.sh"
 	configureCPKubeletScript        = "10-configure-cp-kubelet.sh"
+	configureCalicoScript           = "10-configure-calico.sh"
 	configureClusterAgentPortScript = "10-configure-cluster-agent-port.sh"
 	configureDqlitePortScript       = "10-configure-dqlite-port.sh"
 	configureDqliteAddressScript    = "10-configure-dqlite-address.sh"
@@ -88,12 +90,14 @@ func generateInitStages(cluster clusterplugin.Cluster, token string, userConfig
 	var installCommands []string
 	var upgradeCommands []string
 	installCommands = getBaseInstallCommands(cluster, token, installCommands)
+	calicoConfigCommand := addCalicoConfigCommands(userConfig)
+	installCommands = append(installCommands,fmt.Sprintf("%s %v", calicoConfigCommand, true))
+
 	// figure out endpoint type
 	endpointType := "DNS"
 	if net.ParseIP(cluster.ControlPlaneHost) != nil {
 		endpointType = "IP"
 	}
-	installCommands = append(installCommands, fmt.Sprintf("%s %q %q", scriptPath(configureApiServerScript), endpointType, cluster.ControlPlaneHost))
 
 	if userConfig.ClusterConfiguration.PortCompatibilityRemap {
 		installCommands = append(installCommands, fmt.Sprintf("%s %q", scriptPath(configureClusterAgentPortScript), remappedClusterAgentPort))
@@ -105,14 +109,17 @@ func generateInitStages(cluster clusterplugin.Cluster, token string, userConfig
 
 	// add the bootstrap token
 	installCommands = append(installCommands, fmt.Sprintf("microk8s add-node --token-ttl %v --token %q", tokenTTL, token))
-	installCommands = append(installCommands, scriptPath(configureCPKubeletScript))
 	installCommands = append(installCommands, fmt.Sprintf("%s %v %q", scriptPath(configureDNSScript), userConfig.ClusterConfiguration.UseHostDNS, userConfig.ClusterConfiguration.DNS))
+	installCommands = append(installCommands, fmt.Sprintf("%s %q %q", scriptPath(configureAltNamesScript), endpointType, cluster.ControlPlaneHost))
 
 	addons := parseAddons(userConfig)
 	installCommands = append(installCommands, fmt.Sprintf("%s %s", scriptPath(microk8sEnableScript), strings.Join(addons, " ")))
+	installCommands = append(installCommands, scriptPath(configureCPKubeletScript))
+
 	writeKubeConfigCommand := fmt.Sprintf("%s %s", scriptPath(microk8sKubeConfigScript), userConfig.ClusterConfiguration.WriteKubeconfig)
 	installCommands = append(installCommands, writeKubeConfigCommand)
 	upgradeCommands = append(upgradeCommands, scriptPath(upgradeMicrok8sScript))
+	upgradeCommands = append(upgradeCommands,fmt.Sprintf("%s %v", calicoConfigCommand, false))
 	upgradeCommands = append(upgradeCommands, writeKubeConfigCommand)
 
 	return []yip.Stage{
@@ -129,19 +136,22 @@ func generateInitStages(cluster clusterplugin.Cluster, token string, userConfig
 	}
 }
 
+
+
 func generateControlPlaneJoinStages(cluster clusterplugin.Cluster, token string, userConfig MicroK8sSpec) []yip.Stage {
 	var installCommands []string
 	var upgradeCommands []string
 	var clusterAgentPort string = defaultClusterAgentPort
 
 	installCommands = getBaseInstallCommands(cluster, token, installCommands)
+	calicoConfigCommand := addCalicoConfigCommands(userConfig)
+	installCommands = append(installCommands, fmt.Sprintf("%s %v", calicoConfigCommand, false))
+
 	// figure out endpoint type
 	endpointType := "DNS"
 	if net.ParseIP(cluster.ControlPlaneHost) != nil {
 		endpointType = "IP"
 	}
-	installCommands = append(installCommands, fmt.Sprintf("%s %q %q", scriptPath(configureApiServerScript), endpointType, cluster.ControlPlaneHost))
-
 	if userConfig.ClusterConfiguration.PortCompatibilityRemap {
 		clusterAgentPort = remappedClusterAgentPort
 		installCommands = append(installCommands, fmt.Sprintf("%s %q", scriptPath(configureClusterAgentPortScript), remappedClusterAgentPort))
@@ -150,14 +160,20 @@ func generateControlPlaneJoinStages(cluster clusterplugin.Cluster, token string,
 	if userConfig.ClusterConfiguration.DqliteUseHostIPV4Address {
 		installCommands = append(installCommands, scriptPath(configureDqliteAddressScript))
 	}
-	installCommands = append(installCommands, scriptPath(configureCPKubeletScript))
 	// add join command
 	installCommands = append(installCommands, fmt.Sprintf("%s %q", scriptPath(microk8sJoinScript), fmt.Sprintf("%s:%s/%s", cluster.ControlPlaneHost, clusterAgentPort, token)))
+	//configure after join
+	installCommands = append(installCommands, fmt.Sprintf("%s %q %q", scriptPath(configureAltNamesScript), endpointType, cluster.ControlPlaneHost))
+
 	// add the bootstrap token
 	installCommands = append(installCommands, fmt.Sprintf("microk8s add-node --token-ttl %v --token %q", tokenTTL, token))
+	// label after join
+	installCommands = append(installCommands, scriptPath(configureCPKubeletScript))
 	installCommands = append(installCommands, fmt.Sprintf("%s %s", scriptPath(microk8sKubeConfigScript), userConfig.ClusterConfiguration.WriteKubeconfig))
 
 	upgradeCommands = append(upgradeCommands, scriptPath(upgradeMicrok8sScript))
+	upgradeCommands = append(upgradeCommands,fmt.Sprintf("%s %v", calicoConfigCommand, false))
+
 	return []yip.Stage{
 
 		{
@@ -172,13 +188,18 @@ func generateControlPlaneJoinStages(cluster clusterplugin.Cluster, token string,
 		},
 	}
 }
+
 func generateWorkerJoinStages(cluster clusterplugin.Cluster, token string, userConfig MicroK8sSpec) []yip.Stage {
 
 	var installCommands []string
 	var upgradeCommands []string
 	var clusterAgentPort string = defaultClusterAgentPort
 
 	installCommands = getBaseInstallCommands(cluster, token, installCommands)
+	calicoConfigCommand := addCalicoConfigCommands(userConfig)
+	installCommands = append(installCommands,fmt.Sprintf("%s %v", calicoConfigCommand, false))
+
+
 	if userConfig.ClusterConfiguration.PortCompatibilityRemap {
 		clusterAgentPort = remappedClusterAgentPort
 		installCommands = append(installCommands, fmt.Sprintf("%s %q", scriptPath(configureClusterAgentPortScript), clusterAgentPort))
@@ -187,6 +208,7 @@ func generateWorkerJoinStages(cluster clusterplugin.Cluster, token string, userC
 	installCommands = append(installCommands, fmt.Sprintf("%s %q --worker", scriptPath(microk8sJoinScript), fmt.Sprintf("%s:%s/%s", cluster.ControlPlaneHost, clusterAgentPort, token)))
 
 	upgradeCommands = append(upgradeCommands, scriptPath(upgradeMicrok8sScript))
+
 	return []yip.Stage{
 
 		{
@@ -220,6 +242,20 @@ func getBaseInstallCommands(cluster clusterplugin.Cluster, token string, install
 
 	return installCommands
 }
+func addCalicoConfigCommands(userConfig MicroK8sSpec) string {
+	var calicoConfigCommand string
+
+	if userConfig.ClusterConfiguration.CalicoConfiguration != nil {
+		// need to escape values like cidr for bash scripts
+		var replacer = strings.NewReplacer(
+			`/`, `\/`,
+		)
+		calicoConfig := userConfig.ClusterConfiguration.CalicoConfiguration
+		calicoConfigCommand = fmt.Sprintf("%s %v %q", scriptPath(configureCalicoScript), calicoConfig.CalicoIPinIP, replacer.Replace(calicoConfig.CalicoAutoDetect))
+
+	}
+	return calicoConfigCommand
+}
 func parseAddons(userConfig MicroK8sSpec) []string {
 
 	addons := make([]string, 0, len(userConfig.InitConfiguration.Addons))

scripts/cloudinit/00-install-microk8s.sh

@@ -1,7 +1,10 @@
 #!/bin/bash -xe
 
+packageVersion=$(snap info /opt/microk8s/snaps/microk8s.snap |grep "version:" | awk '{ print $2 }')
+channel=$(echo "${packageVersion%.*}" |cut -f2 -dv)
 snap ack /opt/microk8s/snaps/core.assert
 snap install /opt/microk8s/snaps/core.snap
 snap ack /opt/microk8s/snaps/microk8s.assert
 snap install --classic /opt/microk8s/snaps/microk8s.snap
+snap switch  microk8s  --channel=$channel/stable
 microk8s status --wait-ready

scripts/cloudinit/10-configure-calico.sh

@@ -0,0 +1,36 @@
+#!/bin/bash -xe
+
+# Usage:
+#   $0 true/false <ip autodetect string> <Delete ip pools and ds,only true during init stage>
+#
+# Assumptions:
+#   - microk8s is installed
+#   - calico is installed
+#   - the current node is not part of a cluster (yet)
+
+if [[ "${1}" = "false" && "${2}" = "" ]]; then
+  echo "Will not configure Calico"
+  exit 0
+fi
+
+CNI_YAML="/var/snap/microk8s/current/args/cni-network/cni.yaml"
+microk8s status --wait-ready
+# Stop calico-node and delete ippools to ensure no vxlan pools are left around.True only for first CP node during init install
+if [[ "${3}" == "true" ]]; then
+microk8s kubectl delete daemonset/calico-node -n kube-system || true
+microk8s kubectl delete ippools --all || true
+fi
+# Update cni.yaml manifest for IPIP
+if [[ "${1}" = "true" ]]; then
+sed 's/CALICO_IPV4POOL_VXLAN/CALICO_IPV4POOL_IPIP/' -i "${CNI_YAML}"
+sed 's/calico_backend: "vxlan"/calico_backend: "bird"/' -i "${CNI_YAML}"
+sed 's/-felix-ready/-bird-ready/' -i "${CNI_YAML}"
+sed 's/-felix-live/-bird-live/' -i "${CNI_YAML}"
+fi
+if [[ "${2}" != "" ]]; then
+  sed "s/first-found/${2}/" -i "${CNI_YAML}"
+fi
+
+# Apply the new manifest
+# (TODO): this should perhaps be a touch cni-needs-reload
+microk8s kubectl apply -f "${CNI_YAML}"

scripts/cloudinit/10-configure-cp-kubelet.sh

@@ -2,9 +2,12 @@
 
 #KUBELET_ARGS="${KUBELET_ARGS:-/var/snap/microk8s/current/args/kubelet}"
 
-#sed -i 's/node-labels="\(.*\)"/node-labels="node.kubernetes.io\/controlplane=controlplane,\1"/g' "${KUBELET_ARGS}"
+#vi "${KUBELET_ARGS}"
 
 #snap restart microk8s.daemon-kubelite
-#microk8s status --wait-ready
-NODE_NAME=$(microk8s kubectl get nodes -o json | jq -r .items[].metadata.name)
-microk8s kubectl label nodes $NODE_NAME node-role.kubernetes.io/control-plane=true
+microk8s status --wait-ready
+CP_NODE_NAMES=$(microk8s kubectl get nodes --selector='node.kubernetes.io/microk8s-controlplane' -o json |  jq -r .items[].metadata.name)
+for i in $CP_NODE_NAMES
+do
+microk8s kubectl label nodes $i node-role.kubernetes.io/control-plane=true --overwrite=true
+done

scripts/cloudinit/10-configure-dqlite-address.sh

@@ -10,10 +10,17 @@
 DQLITE="/var/snap/microk8s/current/var/kubernetes/backend"
 
 microk8s status --wait-ready
+while ! NODES_JSON=$(microk8s kubectl get nodes -o json); do
+  echo "Failed querying nodes, retrying"
+done
 
-INTERNAL_IP=$(microk8s kubectl get nodes -o json | jq -r '.items[].status.addresses[] | select(.type=="InternalIP") | .address')
+INTERNAL_IP=$(echo "$NODES_JSON" | jq -r '.items[].status.addresses[] | select(.type=="InternalIP") | .address')
 
 
 grep "Address" "${DQLITE}/info.yaml" | sed "s/127.0.0.1/$INTERNAL_IP/" | tee "${DQLITE}/update.yaml"
 
-snap restart microk8s.daemon-k8s-dqlite
+while ! snap restart microk8s.daemon-k8s-dqlite; do
+  echo "Failed to restart microk8s, will retry"
+  sleep 5
+done
+

scripts/cloudinit/20-microk8s-join.sh

@@ -10,4 +10,15 @@
 while ! microk8s join $@; do
   echo "Failed to join MicroK8s cluster, will retry"
   sleep 5
-done
+done
+# What is this hack? Why do we call snap set here?
+# "snap set microk8s ..." will call the configure hook.
+# The configure hook is where we sanitise arguments to k8s services.
+# When we join a node to a cluster the arguments of kubelet/api-server
+# are copied from the "control plane" node to the joining node.
+# It is possible some deprecated/removed arguments are copied over.
+# For example if we join a 1.24 node to 1.23 cluster arguments like
+# --network-plugin will cause kubelite to crashloop.
+# Threfore we call the conigure hook to clean things.
+# PS. This should be a workaround to a MicroK8s bug.
+snap set microk8s configure=call

types.go

@@ -23,6 +23,9 @@ type ClusterConfiguration struct {
 	UseHostDNS bool `json:"useHostDNS,omitempty"`
 	// Use specified dns server
 	DNS string `json:"dns,omitempty"`
+
+	CalicoConfiguration *CalicoConfiguration `json:"calico,omitempty"`
+
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -44,3 +47,11 @@ type InitConfiguration struct {
 	// List of addons to be enabled upon cluster creation
 	Addons []string `json:"addons,omitempty"`
 }
+
+type CalicoConfiguration struct {
+	// Calico IPinIP
+	CalicoIPinIP bool `json:"calicoIPinIP,omitempty"`
+
+	// Calico Autodetect
+	CalicoAutoDetect string `json:"calicoAutoDetect,omitempty"`
+}