fix: support scaling up/down etcd

builtin/core/playbooks/add_nodes.yaml

@@ -36,7 +36,86 @@
     - etcd
   gather_facts: true
   roles:
-    - etcd
+    - role: etcd
+      when:
+        - .etcd.deployment_type | eq "external"
+
+- hosts:
+    - kube_control_plane
+  tasks:
+    - name: AddNodes | Check if should update apiserver certificates
+      run_once: true
+      add_hostvars:
+        hosts: kube_control_plane
+        vars:
+          need_installed_etcd: >-
+            {{- $needInstalled := list -}}
+            {{- range .groups.etcd -}}
+              {{- if and ((index $.hostvars . "etcd_install_LoadState" "stdout") | eq "not-found") ($.delete_nodes | default list | has . | not) -}}
+                {{- $needInstalled = append $needInstalled . -}}
+              {{- end -}}
+            {{- end -}}
+            {{ $needInstalled | toJson }} 
+    - name: AddNodes | Update apiserver etcd certificates
+      when:
+        - .need_installed_etcd | fromJson | empty | not
+        - .etcd.deployment_type | eq "external"
+      block:
+        - name: AddNodes | Copy etcd CA certificate to control plane node
+          copy:
+            src: >-
+              {{ .etcd.ca_file }}
+            dest: /etc/kubernetes/pki/etcd/ca.crt
+        - name: AddNodes | Copy etcd client certificate to control plane node
+          copy:
+            src: >-
+              {{ .etcd.cert_file }}
+            dest: /etc/kubernetes/pki/etcd/client.crt
+        - name: AddNodes | Copy etcd client key to control plane node
+          copy:
+            src: >-
+              {{ .etcd.key_file }}
+            dest: /etc/kubernetes/pki/etcd/client.key
+        - name: AddNodes | update ks-apiserver
+          command: |
+            {{- $endpoints := list -}}
+            {{- range .groups.etcd | default list -}}
+              {{- $endpoints = append $endpoints (printf "https://%s:2379" (index $.hostvars . "internal_ipv4")) -}}
+            {{- end -}}
+            ETCD_ENDPOINTS="{{ join "," $endpoints }}"
+
+            if ! grep -q 'ClusterConfiguration' /etc/kubernetes/kubeadm-config.yaml 2>/dev/null; then
+              kubectl get cm kubeadm-config -n kube-system -o=jsonpath='{.data.ClusterConfiguration}' > /etc/kubernetes/kubeadm-config.yaml
+            fi
+
+            awk -v ep="$ETCD_ENDPOINTS" '
+              BEGIN {
+                n = split(ep, arr, ",")
+                for (i = 1; i <= n; i++) {
+                  print "      - " arr[i]
+                }
+              }
+            ' > /etc/kubernetes/kubeadm_new_endpoints.yaml
+            # delete old endpoint
+            sed -i '/^[[:space:]]*endpoints:/{
+              :loop
+              N
+              s/\n[[:space:]]\+-.*//; t loop
+              s/\n[[:space:]]*\n/\n/g
+              P
+              D
+            }' /etc/kubernetes/kubeadm-config.yaml
+            # insert new endpoint
+            sed -i "/^[[:space:]]*endpoints:/r /etc/kubernetes/kubeadm_new_endpoints.yaml" /etc/kubernetes/kubeadm-config.yaml        
+            rm /etc/kubernetes/kubeadm_new_endpoints.yaml
+            # update kubeadm-config
+            {{- if .kubernetes.kube_version | semverCompare "<v1.27.0" }}
+            kubeadm config upload from-file --config /etc/kubernetes/kubeadm-config.yaml
+            {{- else }}
+            kubeadm init phase upload-config kubeadm --config /etc/kubernetes/kubeadm-config.yaml
+            {{- end }}
+            # regenerate kube-apiserver
+            kubeadm init phase control-plane apiserver --config /etc/kubernetes/kubeadm-config.yaml
 
 - hosts:
     - k8s_cluster

builtin/core/playbooks/create_cluster.yaml

@@ -35,7 +35,8 @@
 - hosts:
     - etcd
   roles:
-    - etcd
+    - role: etcd
+      when: .etcd.deployment_type | eq "external"
 
 # Install the private image registry
 - hosts:

builtin/core/playbooks/delete_cluster.yaml

@@ -31,8 +31,10 @@
 - hosts:
     - etcd
   roles:
-    - role: uninstall/etcd
-      when: .delete.etcd
+    - role: etcd/scaling_down
+      when:
+        - .delete.etcd
+        - .etcd.deployment_type | eq "external"
 
 - hosts:
    - image_registry

builtin/core/playbooks/delete_nodes.yaml

@@ -9,12 +9,18 @@
   gather_facts: true
   roles:
     - defaults
-    - precheck
+
+- hosts:
+    - etcd
+  roles:
+    - role: etcd
+      when: 
+        - .delete.etcd
+        - .etcd.deployment_type | eq "external"
 
 - hosts:
     - kube_control_plane
-  gather_facts: true
-  tasks:
+  pre_tasks:
     - name: DeleteNode | Ensure at least one control plane node remains in the cluster
       run_once: true
       command: |
@@ -28,6 +34,86 @@
         echo "At least one control plane node must be retained in the cluster." >&2
         exit 1
         {{- end }}
+  tasks:
+    - name: DeleteNode | Update etcd certificate for kube_control_plane
+      when: 
+        - .delete.etcd
+        - .etcd.deployment_type | eq "external"
+      block:
+        - name: DeleteNode | Check if should update apiserver certificates
+          run_once: true
+          add_hostvars:
+            hosts: kube_control_plane
+            vars:
+              need_uninstall_etcd: >-
+                {{- $needUnInstalled := list -}}
+                {{- range .groups.etcd -}}
+                  {{- if $.delete_nodes | default list | has . -}}
+                    {{- $needUnInstalled = append $needUnInstalled . -}}
+                  {{- end -}}
+                {{- end -}}
+                {{ $needUnInstalled | toJson }}  
+        - name: DeleteNode | Update apiserver etcd certificates
+          when:
+            - .need_uninstall_etcd | fromJson | empty | not
+          block:
+            - name: DeleteNode | Copy etcd CA certificate to control plane node
+              copy:
+                src: >-
+                  {{ .etcd.ca_file }}
+                dest: /etc/kubernetes/pki/etcd/ca.crt
+            - name: DeleteNode | Copy etcd client certificate to control plane node
+              copy:
+                src: >-
+                  {{ .etcd.cert_file }}
+                dest: /etc/kubernetes/pki/etcd/client.crt
+            - name: DeleteNode | Copy etcd client key to control plane node
+              copy:
+                src: >-
+                  {{ .etcd.key_file }}
+                dest: /etc/kubernetes/pki/etcd/client.key
+            - name: DeleteNode | update ks-apiserver
+              command: |
+                {{- $endpoints := list -}}
+                {{- range .groups.etcd | default list -}}
+                  {{- if $.need_uninstall_etcd | fromJson | has . | not -}}
+                    {{- $endpoints = append $endpoints (printf "https://%s:2379" (index $.hostvars . "internal_ipv4")) -}}
+                  {{- end -}}
+                {{- end -}}
+                ETCD_ENDPOINTS="{{ join "," $endpoints }}"
+
+                if ! grep -q 'ClusterConfiguration' /etc/kubernetes/kubeadm-config.yaml 2>/dev/null; then
+                  kubectl get cm kubeadm-config -n kube-system -o=jsonpath='{.data.ClusterConfiguration}' > /etc/kubernetes/kubeadm-config.yaml
+                fi
+
+                awk -v ep="$ETCD_ENDPOINTS" '
+                  BEGIN {
+                    n = split(ep, arr, ",")
+                    for (i = 1; i <= n; i++) {
+                      print "      - " arr[i]
+                    }
+                  }
+                ' > /etc/kubernetes/kubeadm_new_endpoints.yaml
+                # delete old endpoint
+                sed -i '/^[[:space:]]*endpoints:/{
+                  :loop
+                  N
+                  s/\n[[:space:]]\+-.*//; t loop
+                  s/\n[[:space:]]*\n/\n/g
+                  P
+                  D
+                }' /etc/kubernetes/kubeadm-config.yaml
+                # insert new endpoint
+                sed -i "/^[[:space:]]*endpoints:/r /etc/kubernetes/kubeadm_new_endpoints.yaml" /etc/kubernetes/kubeadm-config.yaml        
+                rm /etc/kubernetes/kubeadm_new_endpoints.yaml
+                # update kubeadm-config
+                {{- if .kubernetes.kube_version | semverCompare "<v1.27.0" }}
+                kubeadm config upload from-file --config /etc/kubernetes/kubeadm-config.yaml
+                {{- else }}
+                kubeadm init phase upload-config kubeadm --config /etc/kubernetes/kubeadm-config.yaml
+                {{- end }}
+                # regenerate kube-apiserver
+                kubeadm init phase control-plane apiserver --config /etc/kubernetes/kubeadm-config.yaml
 
 - hosts: 
     - k8s_cluster
@@ -67,14 +153,6 @@
         - .delete.dns
         - .delete_nodes | default list | has .inventory_hostname
 
-- hosts:
-    - etcd
-  roles:
-    - role: uninstall/etcd
-      when: 
-        - .delete.etcd
-        - .delete_nodes | default list | has .inventory_hostname
-
 - hosts:
    - image_registry
   roles:

builtin/core/roles/defaults/vars/v1.31.yaml

@@ -93,3 +93,4 @@ image_manifests:
   - quay.io/tigera/operator:v1.34.5
   - docker.io/library/haproxy:2.9.6-alpine
 
+

builtin/core/roles/etcd/tasks/install.yaml → builtin/core/roles/etcd/install/tasks/install.yaml

@@ -12,11 +12,6 @@
       loop:
        - "{{ .etcd.env.data_dir }}"
 
-- name: Install | Generate etcd environment configuration file
-  template:
-    src: etcd.env
-    dest: /etc/etcd.env
-
 - name: Install | Deploy etcd systemd service file
   copy:
     src: etcd.service
@@ -39,3 +34,7 @@
 
 - name: Install | Start and enable etcd systemd service
   command: systemctl daemon-reload && systemctl start etcd && systemctl enable etcd
+
+- name: Install | Refresh etcd.env to set ETCD_INITIAL_CLUSTER_STATE=existing
+  command: sed -i 's/^ETCD_INITIAL_CLUSTER_STATE=new$/ETCD_INITIAL_CLUSTER_STATE=existing/' /etc/etcd.env
+

builtin/core/roles/etcd/install/tasks/main.yaml

@@ -0,0 +1,4 @@
+---
+- include_tasks: install.yaml
+
+- include_tasks: backup_service.yaml

builtin/core/roles/etcd/meta/main.yaml

@@ -0,0 +1,23 @@
+---
+dependencies:
+  - role: etcd/prepare
+
+  - role: etcd/upgrade
+    when: 
+      - .etcd_install_LoadState.stdout | eq "loaded"
+      - .etcd.etcd_version | semverCompare (printf ">v%s" (index .etcd_install_version "stdout" "etcd Version"))
+
+  - role: etcd/scaling_up
+    when:
+      - .installed_etcd | empty | not
+      - .need_installed_etcd | fromJson | empty | not
+
+  - role: etcd/scaling_down
+    when:
+      - .need_uninstall_etcd | fromJson | empty | not
+
+  - role: etcd/install
+    when: 
+      - .etcd_install_LoadState.stdout | eq "not-found"
+      - .need_uninstall_etcd | fromJson | has .inventory_hostname | not
+

builtin/core/roles/etcd/tasks/prepare.yaml → builtin/core/roles/etcd/prepare/tasks/main.yaml

@@ -6,31 +6,39 @@
     fail_msg: >-
       etcd service is installed but not running
 
-- name: Prepare | Set etcd node parameters
+- name: Prepare | Gather etcd node state and membership
   block:
-    - name: Prepare | Identify nodes with installed or missing etcd
+    - name: Prepare | Detect installed, to-install, and to-remove etcd nodes
       run_once: true
       add_hostvars:
         hosts: etcd
         vars:
           installed_etcd: >-
-            {{- $needInstalled := list -}}
+            {{- $installed := list -}}
             {{- range .groups.etcd -}}
-              {{- if (index $.hostvars . "etcd_install_LoadState" "stdout") | eq "loaded" -}}
-                {{- $needInstalled = append $needInstalled . -}}
+              {{- if and ((index $.hostvars . "etcd_install_LoadState" "stdout") | eq "loaded") ($.delete_nodes | default list | has . | not) -}}
+                {{- $installed = append $installed . -}}
               {{- end -}}
             {{- end -}}
-            {{ $needInstalled | first | default "" }}
+            {{ $installed | first | default "" }}
           need_installed_etcd: >-
             {{- $needInstalled := list -}}
             {{- range .groups.etcd -}}
-              {{- if (index $.hostvars . "etcd_install_LoadState" "stdout") | eq "not-found" -}}
+              {{- if and ((index $.hostvars . "etcd_install_LoadState" "stdout") | eq "not-found") ($.delete_nodes | default list | has . | not) -}}
                 {{- $needInstalled = append $needInstalled . -}}
               {{- end -}}
             {{- end -}}
             {{ $needInstalled | toJson }}
+          need_uninstall_etcd: >-
+            {{- $needUnInstalled := list -}}
+            {{- range .groups.etcd -}}
+              {{- if $.delete_nodes | default list | has . -}}
+                {{- $needUnInstalled = append $needUnInstalled . -}}
+              {{- end -}}
+            {{- end -}}
+            {{ $needUnInstalled | toJson }}            
 
-- name: Prepare | Check installed etcd version
+- name: Prepare | Validate installed etcd version
   when: .etcd_install_LoadState.stdout | eq "loaded"
   block:
     - name: Prepare | Ensure target etcd version is not lower than installed version
@@ -40,29 +48,32 @@
         fail_msg: >-
           Installed etcd version: {{ index .etcd_install_version "stdout" "etcd Version" }} is lower than target etcd version: {{ .etcd.etcd_version }}
 
-- name: Prepare | Synchronize etcd package to node if new install or upgrade
-  when:
-    - or (.etcd_install_version.error | empty | not) (.etcd.etcd_version | semverCompare (printf ">v%s" (index .etcd_install_version "stdout" "etcd Version")))
+- name: Prepare | Distribute etcd package for install or upgrade
+  when: >-
+    or 
+      (.etcd_install_version.error | empty | not)
+      (.etcd.etcd_version | semverCompare (printf ">v%s" (index .etcd_install_version "stdout" "etcd Version")))
   block:
-    - name: Prepare | Copy etcd binary package to remote node
+    - name: Prepare | Copy etcd binary package to node
       copy:
         src: >-
           {{ .binary_dir }}/etcd/{{ .etcd.etcd_version }}/{{ .binary_type }}/etcd-{{ .etcd.etcd_version }}-linux-{{ .binary_type }}.tar.gz
         dest: >-
           {{ .tmp_dir }}/etcd-{{ .etcd.etcd_version }}-linux-{{ .binary_type }}.tar.gz
-    - name: Prepare | Extract etcd binary package to /usr/local/bin/
+    - name: Prepare | Extract etcd binaries to /usr/local/bin/
       command: |
         tar --strip-components=1 -C /usr/local/bin/ -xvf {{ .tmp_dir }}/etcd-{{ .etcd.etcd_version }}-linux-{{ .binary_type }}.tar.gz \
           --wildcards 'etcd-{{ .etcd.etcd_version }}-linux-{{ .binary_type }}/etcd*'
 
-- name: Prepare | Synchronize certificates to node for new install or expansion
+- name: Prepare | Synchronize certificates and etcd.env when changed
   when: >-
     or
-      (.etcd_install_version.error | empty | not)
+      (.etcd_install_LoadState.stdout | eq "not-found")
       (and
         (.installed_etcd | empty | not)
         (.need_installed_etcd | fromJson | empty | not)
       )
+      (.need_uninstall_etcd | fromJson | empty | not)
   block:
     - name: Prepare | Copy CA certificate to etcd node
       copy:
@@ -79,3 +90,7 @@
         src: >-
           {{ .etcd.key_file }}
         dest: /etc/ssl/etcd/ssl/server.key
+    - name: Prepare | Render /etc/etcd.env configuration file
+      template:
+        src: etcd.env
+        dest: /etc/etcd.env