MK8S-94: scripts, docs, actions: support etcd distroless

This commit adds support for etcd distroless images for Kubernetes 1.33+.
Above etcd 3.5.21, etcd images are now distroless and upstreamed to the etcd project.

It modifies the backup script to use the correct etcdctl command to snapshot the etcd data instead of using `sh -c` to execute the command in the etcd container.
It modifies the remove node action to use the correct etcdctl command to list and remove etcd members and

It also enforces the use of `=` for the arguments of etcdctl in the disaster recovery documentation, the remove node action and the backup script.

Closes: MK8S-94

Author: ezekiel-alexrod

.github/actions/remove-node/action.yaml

@@ -29,24 +29,26 @@ runs:
       with:
         NODE: ${{ inputs.from-node }}
         COMMAND: >
-          sudo crictl exec -i \"${{ steps.get-container.outputs.RESULT }}\" sh -c \"
-          ETCDCTL_API=3 etcdctl --endpoints https://127.0.0.1:2379
-          --cert /etc/kubernetes/pki/etcd/server.crt
-          --key /etc/kubernetes/pki/etcd/server.key
-          --cacert /etc/kubernetes/pki/etcd/ca.crt
-          member list\" | awk -F ', ' '\$3 ~ \"${{ inputs.node-to-remove }}\" { print \$1 }'
+          sudo crictl exec -i \"${{ steps.get-container.outputs.RESULT }}\"
+          etcdctl
+          --endpoints=https://127.0.0.1:2379
+          --cert=/etc/kubernetes/pki/etcd/server.crt
+          --key=/etc/kubernetes/pki/etcd/server.key
+          --cacert=/etc/kubernetes/pki/etcd/ca.crt
+          member list | awk -F ', ' '\$3 ~ \"${{ inputs.node-to-remove }}\" { print \$1 }'
         CAPTURE_RESULT: "true"
     - name: "Remove the etcd member"
       uses: ./.github/actions/run-command-ssh
       with:
         NODE: ${{ inputs.from-node }}
         COMMAND: >
-          sudo crictl exec -i \"${{ steps.get-container.outputs.RESULT }}\" sh -c \"
-          ETCDCTL_API=3 etcdctl --endpoints https://127.0.0.1:2379
-          --cert /etc/kubernetes/pki/etcd/server.crt
-          --key /etc/kubernetes/pki/etcd/server.key
-          --cacert /etc/kubernetes/pki/etcd/ca.crt
-          member remove ${{ steps.get-id.outputs.RESULT }}\"
+          sudo crictl exec -i \"${{ steps.get-container.outputs.RESULT }}\"
+          etcdctl
+          --endpoints=https://127.0.0.1:2379
+          --cert=/etc/kubernetes/pki/etcd/server.crt
+          --key=/etc/kubernetes/pki/etcd/server.key
+          --cacert=/etc/kubernetes/pki/etcd/ca.crt
+          member remove \"${{ steps.get-id.outputs.RESULT }}\"
     - name: "Remove the node object"
       uses: ./.github/actions/run-command-ssh
       with:

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## Release 133.0.0 (in development)
 
+### Enhancements
+
+- Support etcd distroless images for Kubernetes 1.33+. Above etcd 3.5.21, etcd images are now distroless and upstreamed to the etcd project.
+  (PR[#4740](https://github.com/scality/metalk8s/pull/4740))
+
 ## Release 132.0.0 (in development)
 
 ### Enhancements

docs/operation/disaster_recovery/bootstrap_backup_restore.rst

@@ -56,21 +56,21 @@ Restoring a Bootstrap Node
       .. code::
 
          crictl exec -it "$CONT_ID" \
-            etcdctl --endpoints https://localhost:2379 \
-            --cacert /etc/kubernetes/pki/etcd/ca.crt \
-            --key /etc/kubernetes/pki/etcd/server.key \
-            --cert /etc/kubernetes/pki/etcd/server.crt \
+            etcdctl --endpoints=https://localhost:2379 \
+            --cacert=/etc/kubernetes/pki/etcd/ca.crt \
+            --key=/etc/kubernetes/pki/etcd/server.key \
+            --cert=/etc/kubernetes/pki/etcd/server.crt \
             member list
 
    #. Remove the etcd member (replace ``<etcd_id>`` in the command).
 
       .. code::
 
          crictl exec -it "$CONT_ID" \
-            etcdctl --endpoints https://localhost:2379 \
-            --cacert /etc/kubernetes/pki/etcd/ca.crt \
-            --key /etc/kubernetes/pki/etcd/server.key \
-            --cert /etc/kubernetes/pki/etcd/server.crt \
+            etcdctl --endpoints=https://localhost:2379 \
+            --cacert=/etc/kubernetes/pki/etcd/ca.crt \
+            --key=/etc/kubernetes/pki/etcd/server.key \
+            --cert=/etc/kubernetes/pki/etcd/server.crt \
             member remove <etcd_id>
 
 #. Because multiple bootstrap nodes are not supported, remove the old

scripts/backup.sh.in

@@ -115,21 +115,23 @@ backup_cas() {
 
 backup_etcd() {
     local -r etcd_snapshot="etcd_snapshot_$(date -u +%Y%m%d_%H%M%S)"
-    local -r cmd=(
-        "ETCDCTL_API=3 etcdctl --endpoints https://127.0.0.1:2379"
-        "--cert /etc/kubernetes/pki/etcd/salt-master-etcd-client.crt"
-        "--key /etc/kubernetes/pki/etcd/salt-master-etcd-client.key"
-        "--cacert /etc/kubernetes/pki/etcd/ca.crt"
-        "snapshot save $etcd_snapshot"
-    )
     local etcd_container=''
     echo "Snapshot etcd"
     etcd_container="$(crictl ps -q \
         --label io.kubernetes.pod.namespace=kube-system \
         --label io.kubernetes.container.name=etcd \
         --state Running)"
-    echo "Running '${cmd[*]}' in etcd container $etcd_container"
-    crictl exec -i "$etcd_container" sh -c "${cmd[*]}"
+    echo "Running etcdctl snapshot save in etcd container $etcd_container"
+    # Note: etcd image in Kubernetes 1.33+ is distroless (no shell), so we must
+    # exec etcdctl directly without using "sh -c".
+    # etcd 3.4+ defaults to API v3, so ETCDCTL_API=3 is no longer required.
+    crictl exec -i "$etcd_container" \
+        etcdctl \
+        --endpoints=https://127.0.0.1:2379 \
+        --cert=/etc/kubernetes/pki/etcd/salt-master-etcd-client.crt \
+        --key=/etc/kubernetes/pki/etcd/salt-master-etcd-client.key \
+        --cacert=/etc/kubernetes/pki/etcd/ca.crt \
+        snapshot save "$etcd_snapshot"
 
     local -r rootfs_v1="/run/containerd/io.containerd.runtime.v1.linux/k8s.io/${etcd_container}/rootfs"
     local -r rootfs_v2="/run/containerd/io.containerd.runtime.v2.task/k8s.io/${etcd_container}/rootfs"