fix(engine): enable support for high-availability via etcd

Author: nicklasfrahm

README.md

@@ -6,6 +6,8 @@
 
 A lightweight Kubernetes engine that deploys `k3s` clusters declaratively based on a cluster configuration file. The name is an abbreviation for _k3s engine_ and a hommage to the German word for cheese, _Käse [ˈkɛːzə]_.
 
+**🐉 Here be dragons:** As the version indicates, this is experimental software. The API is subject to change and there are no stability guarantees. Use at your own risk. We recommend to have a disaster recovery strategy in place.
+
 ## Quickstart 💡
 
 If you want to test `k3se` you can use [Vagrant][website-vagrant]. All examples in the `examples/` folder can be used with the provided `Vagrantfile` that provisions 3 Ubuntu VMs. To bring up the VMs you can run the following command:
@@ -32,11 +34,10 @@ $ echo "$(whoami) ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/$(whoami)
 
 The following features are currently not supported, but are planned for future releases:
 
-- **High availability**  
-  Currently the installation will fail if more than a single node with the role `server` is specified.
-
 - **Downsizing**  
-  If nodes are removed from the cluster configuration, they are not decommissioned. We plan to enable this feature in the future using the `git` history of the cluster configuration.
+  If nodes are removed from the cluster configuration, they are not decommissioned. We plan to enable this feature in the future; performing operations similar to `kubectl cordon` and `kubectl drain` automagically.
+- **Diffing**  
+  Using the `git` history of the cluster configuration to display potential actions that can be taken to bring the cluster up to date with the configuration.
 
 ## License 📄
 

examples/agents.yml

@@ -6,9 +6,13 @@ version: stable
 # to all nodes in the cluster. All options are equivalent to the
 # commmand line options of the `k3s` command.
 cluster:
+  # It is highly recommended to always specify this option as it
+  # is used to determine the server URL of the cluster.
   tls-san:
     - 192.168.56.11
   write-kubeconfig-mode: "644"
+  node-label:
+    - example=agents
 
 # A list of all nodes in the cluster and their connection information.
 nodes:
@@ -19,20 +23,22 @@ nodes:
       key-file: ~/.ssh/id_ed25519
     config:
       node-label:
-        - site=vagrant
+        - hostname=kube1
+
   - role: agent
     ssh:
       host: 192.168.56.12
       user: vagrant
       key-file: ~/.ssh/id_ed25519
     config:
       node-label:
-        - site=vagrant
+        - hostname=kube2
+
   - role: agent
     ssh:
       host: 192.168.56.13
       user: vagrant
       key-file: ~/.ssh/id_ed25519
     config:
       node-label:
-        - site=vagrant
+        - hostname=kube3

examples/ha.yml

@@ -0,0 +1,59 @@
+# Version may either be a specific k3s version or a release channel
+# as listed here: https://update.k3s.io/v1-release/channels
+version: stable
+
+# Cluster provides cluster-wide settings that should be applied
+# to all nodes in the cluster. All options are equivalent to the
+# commmand line options of the `k3s` command.
+cluster:
+  # It is highly recommended to always specify this option as it
+  # is used to determine the server URL of the cluster.
+  tls-san:
+    - 192.168.56.11
+  write-kubeconfig-mode: "644"
+  # IMPORTANT: Setting this options is required if your
+  # HA cluster nodes have multiple network interfaces.
+  flannel-iface: "enp0s3"
+  node-label:
+    - example=ha
+
+# A list of all nodes in the cluster and their connection information.
+nodes:
+  - role: server
+    ssh:
+      host: 192.168.56.11
+      user: vagrant
+      key-file: ~/.ssh/id_ed25519
+    config:
+      node-label:
+        - mylabel=a
+      # IMPORTANT: Setting this options is required if your
+      # HA cluster nodes have multiple network interfaces.
+      node-ip: 192.168.56.11
+      node-external-ip: 192.168.56.11
+
+  - role: server
+    ssh:
+      host: 192.168.56.12
+      user: vagrant
+      key-file: ~/.ssh/id_ed25519
+    config:
+      node-label:
+        - mylabel=b
+      # IMPORTANT: Setting this options is required if your
+      # HA cluster nodes have multiple network interfaces.
+      node-ip: 192.168.56.12
+      node-external-ip: 192.168.56.12
+
+  - role: server
+    ssh:
+      host: 192.168.56.13
+      user: vagrant
+      key-file: ~/.ssh/id_ed25519
+    config:
+      node-label:
+        - mylabel=c
+      # IMPORTANT: Setting this options is required if your
+      # HA cluster nodes have multiple network interfaces.
+      node-ip: 192.168.56.13
+      node-external-ip: 192.168.56.13

examples/proxy.yml

@@ -6,9 +6,13 @@ version: stable
 # to all nodes in the cluster. All options are equivalent to the
 # commmand line options of the `k3s` command.
 cluster:
+  # It is highly recommended to always specify this option as it
+  # is used to determine the server URL of the cluster.
   tls-san:
     - 192.168.56.11
   write-kubeconfig-mode: "644"
+  node-label:
+    - example=proxy
 
 # A list of all nodes in the cluster and their connection information.
 nodes:
@@ -19,7 +23,7 @@ nodes:
       key-file: ~/.ssh/id_ed25519
     config:
       node-label:
-        - site=vagrant
+        - mylabel=a
 
 # An SSH proxy, also known as jumpbox or a bastion host
 # can be used to access nodes in a private network.

examples/standalone.yml

@@ -6,9 +6,13 @@ version: stable
 # to all nodes in the cluster. All options are equivalent to the
 # commmand line options of the `k3s` command.
 cluster:
+  # It is highly recommended to always specify this option as it
+  # is used to determine the server URL of the cluster.
   tls-san:
     - 192.168.56.11
   write-kubeconfig-mode: "644"
+  node-label:
+    - example=standalone
 
 # A list of all nodes in the cluster and their connection information.
 nodes:
@@ -19,4 +23,4 @@ nodes:
       key-file: ~/.ssh/id_ed25519
     config:
       node-label:
-        - site=vagrant
+        - mylabel=a

pkg/engine/config.go

@@ -33,7 +33,10 @@ type K3sConfig struct {
 	TLSSAN              []string `yaml:"tls-san,omitempty"`
 	Disable             []string `yaml:"disable,omitempty"`
 	AdvertiseAddress    string   `yaml:"advertise-address,omitempty"`
-	NodeLabel           []string `yaml:"node-label"`
+	FlannelIface        string   `yaml:"flannel-iface,omitempty"`
+	NodeIP              string   `yaml:"node-ip,omitempty"`
+	NodeExternalIP      string   `yaml:"node-external-ip,omitempty"`
+	NodeLabel           []string `yaml:"node-label,omitempty"`
 	// TODO: Add missing config options as specified here:
 	//       https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/#k3s-server-cli-help
 }
@@ -118,10 +121,8 @@ func (c *Config) Verify() error {
 		return errors.New("no control-plane nodes specified")
 	}
 
-	if controlPlanes > 1 {
-		return errors.New("unimplemented: multiple control-plane nodes")
-
-		// TODO: Check that backend is not SQLite if HA is enabled.
+	if controlPlanes%2 == 0 {
+		return errors.New("number of control-plane nodes must be odd")
 	}
 
 	return nil