fix(e2e): wait for RAFT cluster convergence in HA db corruption test (#6348)

After OVN database corruption recovery, the RAFT cluster may not
immediately show all servers in `cluster/status` output even though
pods are ready and db files are healthy. This causes intermittent
failures in `getDbSidsFromClusterStatus()` when it asserts the server
count right after recovery.

Replace the immediate assertion with a WaitUntil poll (up to 30s) that
waits for all pods to report the expected number of servers before
proceeding with validation.

Signed-off-by: Mengxin Liu <liumengxinfly@gmail.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: oilbeater

test/e2e/ha/ha_test.go

@@ -131,18 +131,33 @@ func getDbSidsFromClusterStatus(f *framework.Framework, deploy *appsv1.Deploymen
 	framework.ExpectNoError(err)
 	framework.ExpectHaveLen(pods.Items, int(*deploy.Spec.Replicas))
 
+	expectedCount := len(pods.Items)
 	dbServers := make(map[string]map[string]string)
 	for _, db := range [...]string{"nb", "sb"} {
-		ginkgo.By("Getting ovn" + db + " db server ids on all ovn-central pods")
+		ginkgo.By("Waiting for ovn" + db + " db cluster to show all servers on every ovn-central pod")
 		for pod := range slices.Values(pods.Items) {
-			stdout, stderr, err := framework.ExecShellInPod(context.Background(), f, pod.Namespace, pod.Name, cmdClusterStatus(db))
-			framework.ExpectNoError(err, fmt.Sprintf("failed to get ovn%s db status in pod %s: stdout = %q, stderr = %q", db, pod.Name, stdout, stderr))
-			status := parseClusterStatus(stdout)
-			framework.ExpectHaveLen(status.Servers, len(pods.Items), "unexpected number of servers in ovn%s db status in pod %s: stdout = %q, stderr = %q", db, pod.Name, stdout, stderr)
+			var lastStdout, lastStderr string
+			framework.WaitUntil(2*time.Second, 30*time.Second, func(_ context.Context) (bool, error) {
+				stdout, stderr, err := framework.ExecShellInPod(context.Background(), f, pod.Namespace, pod.Name, cmdClusterStatus(db))
+				if err != nil {
+					return false, nil
+				}
+				lastStdout, lastStderr = stdout, stderr
+				var count int
+				for line := range strings.SplitSeq(stdout, "\n") {
+					if slices.Contains(strings.Fields(line), "at") {
+						count++
+					}
+				}
+				return count == expectedCount, nil
+			}, fmt.Sprintf("ovn%s db on pod %s to show %d servers", db, pod.Name, expectedCount))
+
+			status := parseClusterStatus(lastStdout)
+			framework.ExpectHaveLen(status.Servers, expectedCount, "unexpected number of servers in ovn%s db status in pod %s: stdout = %q, stderr = %q", db, pod.Name, lastStdout, lastStderr)
 			if len(dbServers[db]) == 0 {
 				dbServers[db] = maps.Clone(status.Servers)
 			} else {
-				framework.ExpectEqual(status.Servers, dbServers[db], "inconsistent servers in ovn%s db status in pod %s: stdout = %q, stderr = %q", db, pod.Name, stdout, stderr)
+				framework.ExpectEqual(status.Servers, dbServers[db], "inconsistent servers in ovn%s db status in pod %s: stdout = %q, stderr = %q", db, pod.Name, lastStdout, lastStderr)
 			}
 		}
 	}