Debug e2e

.github/workflows/e2e.yaml

@@ -60,13 +60,26 @@ jobs:
     strategy:
       matrix:
         ginkgo_focus:
-          - "KCP remediation"
-          - "MachineDeployment remediation"
-          - "Workload cluster creation"
-          - "Workload cluster scaling"
-          - "Workload cluster upgrade"
           - "Certificate Refresh"
-          - "Orchestrated In place upgrades"
+          - "Certificate Refresh"
+          - "Certificate Refresh"
+          - "Certificate Refresh"
+          - "Certificate Refresh"
+          - "Certificate Refresh"
+          - "Certificate Refresh"
+          - "Certificate Refresh"
+          - "Certificate Refresh"
+          - "Certificate Refresh"
+          - "Workload cluster scaling"
+          - "Workload cluster scaling"
+          - "Workload cluster scaling"
+          - "Workload cluster scaling"
+          - "Workload cluster scaling"
+          - "Workload cluster scaling"
+          - "Workload cluster scaling"
+          - "Workload cluster scaling"
+          - "Workload cluster scaling"
+          - "Workload cluster scaling"
       # TODO(ben): Remove once all tests are running stable.
       fail-fast: false
     steps:
@@ -111,7 +124,28 @@ jobs:
           sudo sysctl fs.inotify.max_user_instances=8192
       - name: Run e2e tests
         run: |
+          set +e
+          ./pebble-follower.sh &
           sudo GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e
+          exit_code="$?"
+
+          # Stop log collectors.
+          ps aux | grep "docker exec" | awk '{print $2}' | xargs kill
+
+          tar -czvf test-logs.tar.gz docker-ps.txt logs
+
+          # Generate a random string for the artefact name.
+          echo "RANDOM_STRING=$(tr -dc A-Za-z0-9 </dev/urandom | head -c 13)" >> $GITHUB_ENV
+          exit $exit_code
+
+      - name: Upload test service logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-test-logs-${{ github.run_id }}-${{ env.RANDOM_STRING }}
+          path: |
+            test-logs.tar.gz
+
       - name: Change artifact permissions
         if: always()
         run: |

pebble-follower.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -x
+
+services="containerd k8s-dqlite k8sd kube-apiserver kube-controller-manager kube-proxy kube-scheduler kubelet"
+
+mkdir -p logs
+
+until [ `docker ps -a | grep "control-plane-" | wc -l` = "6" ]; do
+  echo "Waiting for control plane nodes...";
+  sleep 15
+done
+
+# k8s services may still need to be initialized before we start collecting logs.
+sleep 30
+
+docker ps -a > docker-ps.txt
+
+while read -r container; do
+  container_name="${container##* }"
+  mkdir logs/$container_name
+
+  for service in $services; do
+    nohup docker exec $container_name pebble logs -f -n 1000000 $service >> logs/$container_name/$service.log &
+  done
+done < <(docker ps -a | grep control-plane-)