CLOUDP-383543: fix flaky secondaries check using replSetGetStatus (#824)

## Summary

`client.secondaries` in pymongo relies on async background topology
discovery — after `ismaster` initiates the connection, monitor threads
continue discovering replica set members in the background. This means
`client.secondaries` can return an empty or incomplete set immediately
after connecting, causing intermittent test failures.

This PR replaces all usages of `client.secondaries` with a new
`KubernetesTester.get_replica_set_secondaries()` helper that runs
`replSetGetStatus` on the primary. This is a synchronous call that
returns the authoritative replica set state directly from the server,
bypassing the client-side topology cache entirely.

## Proof of Work

N/A — test-only change fixing flaky assertions.

## Checklist

- [x] Have you linked a jira ticket and/or is the ticket in the title?
- [ ] Have you checked whether your jira ticket required DOCSP changes?
- [x] Have you added changelog file?
    - use `skip-changelog` label if not needed

Author: nammn

docker/mongodb-kubernetes-tests/kubetester/kubetester.py

@@ -1334,7 +1334,7 @@ def yield_existing_csrs(csr_names, timeout=300):
         return yield_existing_csrs(csr_names, timeout)
 
     @staticmethod
-    def get_populated_mongo_client(hosts: list[str], ssl: bool = False) -> pymongo.MongoClient:
+    def get_connected_mongo_client(hosts: list[str], ssl: bool = False) -> pymongo.MongoClient:
         mongodburi = KubernetesTester.build_mongodb_uri_for_rs(hosts)
         options = {}
         if ssl:
@@ -1346,6 +1346,16 @@ def get_populated_mongo_client(hosts: list[str], ssl: bool = False) -> pymongo.M
 
         return client
 
+    @staticmethod
+    def get_replica_set_secondaries(client: pymongo.MongoClient) -> list:
+        """Returns healthy secondaries queried from the primary via replSetGetStatus.
+
+        Prefer this over client.secondaries, which relies on pymongo's async topology
+        discovery and may return an incomplete result immediately after connecting.
+        """
+        status = client.admin.command("replSetGetStatus", read_preference=pymongo.ReadPreference.PRIMARY)
+        return [m for m in status["members"] if m["stateStr"] == "SECONDARY" and m["health"] == 1]
+
     def _get_pods(self, podname, qty=3):
         return [podname.format(i) for i in range(qty)]
 

docker/mongodb-kubernetes-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_geo_sharding.py

@@ -149,5 +149,5 @@ def test_assert_shard_primary_replicas(self, sc: MongoDB):
         }
         for shard_idx, cluster_idx in cluster_primary_member_mapping.items():
             shard_primary_hostname = sc.shard_hostname(shard_idx, 0, cluster_idx)
-            client = KubernetesTester.get_populated_mongo_client(hosts=[shard_primary_hostname])
+            client = KubernetesTester.get_connected_mongo_client(hosts=[shard_primary_hostname])
             assert client.is_primary

docker/mongodb-kubernetes-tests/tests/replicaset/replica_set_pv.py

@@ -91,10 +91,10 @@ def test_replica_set_was_configured(self):
         "Should connect to one of the mongods and check the replica set was correctly configured."
         hosts = ["rs001-pv-{}.rs001-pv-svc.{}.svc.cluster.local:27017".format(i, self.namespace) for i in range(3)]
 
-        client = self.get_populated_mongo_client(hosts=hosts)
+        client = self.get_connected_mongo_client(hosts=hosts)
 
         assert client.primary is not None
-        assert len(client.secondaries) == 2
+        assert len(KubernetesTester.get_replica_set_secondaries(client)) == 2
 
 
 @pytest.mark.e2e_replica_set_pv

docker/mongodb-kubernetes-tests/tests/shardedcluster/sharded_cluster.py

@@ -130,9 +130,9 @@ def test_both_shards_are_configured(self, sc: MongoDB):
                     hosts.append(hostname)
 
             logger.debug(f"Checking for connectivity of hosts: {hosts}")
-            client = KubernetesTester.get_populated_mongo_client(hosts=hosts)
+            client = KubernetesTester.get_connected_mongo_client(hosts=hosts)
             assert client.primary is not None
-            assert len(client.secondaries) == 2
+            assert len(KubernetesTester.get_replica_set_secondaries(client)) == 2
 
     def test_monitoring_versions(self, sc: MongoDB):
         """Verifies that monitoring agent is configured for each process in the deployment"""

docker/mongodb-kubernetes-tests/tests/shardedcluster/sharded_cluster_pv.py

@@ -96,10 +96,10 @@ def test_shard0_was_configured(self, sc: MongoDB):
                 hostname = sc.shard_hostname(0, member_idx, cluster_member_client.cluster_index)
                 hosts.append(hostname)
 
-        client = KubernetesTester.get_populated_mongo_client(hosts=hosts)
+        client = KubernetesTester.get_connected_mongo_client(hosts=hosts)
 
         assert client.primary is not None
-        assert len(client.secondaries) == 2
+        assert len(KubernetesTester.get_replica_set_secondaries(client)) == 2
 
     def test_pvc_are_bound(self, sc: MongoDB):
         for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace):