Merge branch 'bugfix/fix-flaky-on-apiserver-restart' into tmp/octopus/w/131.0/bugfix/fix-flaky-on-apiserver-restart

Author: bert-e

.github/actions/sosreport-logs/action.yaml

@@ -24,11 +24,11 @@ runs:
       run: |
         for host in ${HOSTS_LIST}; do
           ssh -F ssh_config ${host} \
-          "sudo sosreport --all-logs \
+          "sudo sos report --all-logs \
           -o metalk8s -kmetalk8s.k8s-resources -kmetalk8s.pod-logs -k metalk8s.describe -k metalk8s.metrics \
           -o metalk8s_containerd -kmetalk8s_containerd.all -kmetalk8s_containerd.logs \
           --batch --tmp-dir /var/tmp && \
-          sudo chown -R 1000:1000 /var/tmp/sosreport*"
+          sudo chown 1000:1000 /var/tmp/sosreport*"
         done
     - name: collect sosreport on each node
       shell: bash

.github/scripts/stabilize_snapshot.py

@@ -205,7 +205,7 @@ def check_pods_running():
             "--all",
             "--all-namespaces",
             "--for=condition=Ready",
-            "--timeout=10s",
+            "--timeout=60s",
             "--selector=!job-name",  # We filter out Jobs (they can't be Ready)
             capture_output=True,
             check=True,

CHANGELOG.md

@@ -9,6 +9,11 @@
 - Add 1 second request interval on every salt call using http.wait_for_successful_query
   (PR[#4609](https://github.com/scality/metalk8s/pull/4609))
 
+### Bug fixes
+
+- Make sure the apiserver is running after reconfiguring the pod
+  (PR[#4611](https://github.com/scality/metalk8s/pull/4611))
+
 ## Release 130.0.0
 
 ### Enhancements

docs/operation/cluster_monitoring/prometheus.rst

@@ -20,7 +20,7 @@ To generate a snapshot, use the
 
 .. code-block:: console
 
-   root@host # sosreport --batch --build -o metalk8s -kmetalk8s.prometheus-snapshot=True
+   root@host # sos report --batch --build -o metalk8s -kmetalk8s.prometheus-snapshot=True
 
 The name of the generated archive is printed on the console output and
 the Prometheus snapshot can be found under ``prometheus_snapshot`` directory.

docs/operation/sosreport.rst

@@ -19,7 +19,7 @@ To include logs and configuration for containerd and MetalK8s components, run:
 
 .. code-block:: console
 
-   root@your-machine # sosreport --batch --all-logs \
+   root@your-machine # sos report --batch --all-logs \
        -o metalk8s -kmetalk8s.k8s-resources -kmetalk8s.pod-logs -kmetalk8s.describe -kmetalk8s.metrics \
        -o metalk8s_containerd -kmetalk8s_containerd.all -kmetalk8s_containerd.logs
 
@@ -37,4 +37,4 @@ To display the full list of available plugins and their options, run:
 
 .. code-block:: shell
 
-   sosreport --list-plugins
+   sos report --list-plugins

salt/metalk8s/kubernetes/apiserver/installed.sls

@@ -163,16 +163,27 @@ Restart kubelet to make it pick up the manifest changes:
         timeout: 120
         raise_on_timeout: false
     - require_in:
-      - module: Make sure kube-apiserver container is up and ready
+      - module: Delay after kube-apiserver Pod deployment
 
 {%- endif %}
 
+# This is a bit ugly but currently we have no easy way to find if the
+# pod has been updated already or not, so we need to wait a bit
+Delay after kube-apiserver Pod deployment:
+  module.wait:
+    - test.sleep:
+      - length: 10
+    - watch:
+      - metalk8s: Create kube-apiserver Pod manifest
+
 Make sure kube-apiserver container is up and ready:
   module.run:
     - cri.wait_container:
       - name: kube-apiserver
       - state: running
       - timeout: 120
+    - require:
+      - module: Delay after kube-apiserver Pod deployment
     - onchanges:
       - metalk8s: Create kube-apiserver Pod manifest
   http.wait_for_successful_query:

salt/metalk8s/orchestrate/apiserver.sls

@@ -38,6 +38,7 @@ Deploy apiserver {{ node }} to {{ dest_version }}:
     - tgt: {{ node }}
     - sls:
       - metalk8s.kubernetes.apiserver
+      - metalk8s.kubernetes.apiserver-proxy
     - saltenv: metalk8s-{{ dest_version }}
     - require:
       - salt: Check pillar on {{ node }}