Adding cbcollect inbetween the columnar volume test

Change-Id: I056fb934217c972c4bcc0700d9528e06fbcdb0e5
Reviewed-on: https://review.couchbase.org/c/TAF/+/226551
Reviewed-by: Thuan Nguyen <soccon@gmail.com>
Tested-by: Ritesh Agarwal <ritesh.agarwal@couchbase.com>

Author: ritalrw

couchbase_utils/capella_utils/dedicated.py

@@ -631,9 +631,8 @@ def trigger_log_collection(pod, tenant, cluster_id, log_id=""):
                                  tenant.user,
                                  tenant.pwd,
                                  pod.TOKEN)
-        log_id = {"ticketId": log_id}
         resp = capella_api.trigger_log_collection(cluster_id,
-                                                  log_id=log_id)
+                                                  ticketId=log_id)
         if resp.status_code != 201:
             CapellaUtils.log.critical("Logs collection failed:{}".
                                       format(resp.status_code))

pytests/aGoodDoctor/goldfish/CbasUtil.py

@@ -216,9 +216,8 @@ def wait_for_link_connect(self, cluster, link_name, timeout=3600):
     def discharge_CBAS(self):
         self.stop_run = True
 
-    def wait_for_ingestion(self, cluster, databases, timeout=86400):
+    def wait_for_ingestion(self, cluster, databases, timeout=86400, ingestion_result=list()):
         client = cluster.SDKClients[0].cluster
-        _results = list()
         def check_in_th(collection, items):
             status = False
             stop_time = time.time() + timeout
@@ -271,7 +270,7 @@ def check_in_th(collection, items):
                     traceback.print_exc()
                 time.sleep(random.randint(60,120))
             if status is False:
-                _results.append(status)
+                ingestion_result.append(status)
         ths = list()
         for database in databases:
             for collection in database.cbas_collections:
@@ -282,7 +281,7 @@ def check_in_th(collection, items):
                 ths.append(th)
         for th in ths:
             th.join()
-        return False not in _results
+        return False not in ingestion_result
 
 
 class CBASQueryLoad:

pytests/aGoodDoctor/goldfish/GoldfishVolume.py

@@ -303,20 +303,23 @@ def test_rebalance(self):
         while loop > 0:
             loop -= 1
             self.ingestion_ths = list()
+            ingestion_result = list()
 
             # Create new collections
             for dataSource in self.data_sources["remoteCouchbase"] + self.data_sources["mongo"]:
                 for tenant in self.tenants:
                     for columnar in tenant.columnar_instances:
                         self.drCBAS.disconnect_link(columnar, dataSource.link_name)
                         self.sleep(60, "wait after previous link disconnect")
+                        self.cbcollect_logs(tenant, cluster.cluster_id, log_id="1")
                         dataSource.link_name = "{}_".format(dataSource.type) + ''.join([random.choice(string.ascii_letters + string.digits) for _ in range(5)])
                         dataSource.links.append(dataSource.link_name)
                         dataSource.loadDefn.get("cbas")[1] = dataSource.loadDefn.get("cbas")[1] + self.default_workload.get("cbas")[1]
                         self.drCBAS.create_links(columnar, [dataSource])
                         th = threading.Thread(
                             target=self.drCBAS.wait_for_ingestion,
-                            args=(columnar, [dataSource], self.index_timeout))
+                            args=(columnar, [dataSource], self.index_timeout,
+                                  ingestion_result))
                         th.start()
                         self.ingestion_ths.append(th)
             if scaling:
@@ -357,6 +360,10 @@ def test_rebalance(self):
                     self.sleep(1800, "Lets the ingestion/query running for 30 mins post scaling")
             for th in self.ingestion_ths:
                 th.join()
+            if False in ingestion_result:
+                for tenant in self.tenants:
+                    for columnar in tenant.columnar_instances:
+                        self.cbcollect_logs(tenant, cluster.cluster_id, log_id="2")
 
         for ql in self.cbasQL:
             ql.stop_query_load()

pytests/aGoodDoctor/hostedOPD.py

@@ -7,6 +7,7 @@
 
 from com.couchbase.test.taskmanager import TaskManager
 from com.couchbase.test.sdk import Server, SDKClientPool
+from table_view import TableView
 from opd import OPD
 from membase.api.rest_client import RestConnection
 import random
@@ -301,3 +302,13 @@ def wait_for_rebalances(self, rebalance_tasks):
         for task in rebalance_tasks:
             self.task_manager.get_task_result(task)
             self.assertTrue(task.result, "Cluster Upgrade Failed...")
+
+    def cbcollect_logs(self, tenant, cluster_id, log_id=""):
+        CapellaUtils.trigger_log_collection(self.pod, tenant, cluster_id,
+                                            log_id=log_id)
+        table = TableView(self.log.info)
+        table.add_row(["URL"])
+        task = CapellaUtils.check_logs_collect_status(self.pod, tenant, cluster_id)
+        for _, logInfo in sorted(task["perNode"].items()):
+            table.add_row([logInfo["url"]])
+        table.display("Cluster: {}".format(cluster_id))