Adding cbcontbk and cbbackupmgr log collection if test fails

Change-Id: Id5814b0ea414176a0b44f708f3fee5fb155c3693
Reviewed-on: https://review.couchbase.org/c/TAF/+/244657
Tested-by: Build Bot <build@couchbase.com>
Reviewed-by: Ashwin <ashwin.govindarajulu@couchbase.com>

Author: raghavsake

couchbase_utils/cb_tools/cbbackupmgr.py

@@ -81,12 +81,12 @@ def backup(self, archive_dir, repo_name, cluster_host=None,
         cmd += self.cli_flags
         self.log.debug(f"Executing command: {cmd}")
         output, error = self._execute_cmd(cmd)
-        
+
         self.log.debug(f"Command output: {output}")
-            
+
         if not output or error:
             self.log.error(f"Command failed with error: {error}")
-            
+
         return output, error
 
     """
@@ -102,15 +102,15 @@ def create_repo(self, archive_dir, repo_name):
         """
         cmd = "%s config --archive %s --repo %s" % (
             self.cbstatCmd, archive_dir, repo_name)
-            
+
         self.log.debug(f"Executing command: {cmd}")
         output, error = self._execute_cmd(cmd)
-        
+
         self.log.debug(f"Command output: {output}")
-            
+
         if not output or error:
             self.log.error(f"Command failed with error: {error}")
-            
+
         return output, error
 
     """
@@ -127,15 +127,15 @@ def list_backups(self, archive_dir, repo_name):
         cmd = "%s list --archive %s --repo %s" % (
             self.cbstatCmd, archive_dir, repo_name)
         cmd += self.cli_flags
-        
+
         self.log.debug(f"Executing command: {cmd}")
         output, error = self._execute_cmd(cmd)
-        
+
         self.log.debug(f"Command output: {output}")
-            
+
         if not output or error:
             self.log.error(f"Command failed with error: {error}")
-            
+
         return output, error
 
     """
@@ -213,15 +213,15 @@ def restore(self, archive_dir, repo_name, cluster_host=None,
             cmd += " --filter-values %s" % filter_values
 
         cmd += self.cli_flags
-        
+
         self.log.debug(f"Executing command: {cmd}")
         output, error = self._execute_cmd(cmd)
-        
+
         self.log.debug(f"Command output: {output}")
-            
+
         if not output or error:
             self.log.error(f"Command failed with error: {error}")
-            
+
         return output, error
 
     """
@@ -238,18 +238,18 @@ def remove(self, archive_dir, repo_name, backup_range=None):
         """
         cmd = "%s remove --archive %s --repo %s" % (
             self.cbstatCmd, archive_dir, repo_name)
-        
+
         if backup_range:
             cmd += " --backups %s" % backup_range
-        
+
         self.log.debug(f"Executing command: {cmd}")
         output, error = self._execute_cmd(cmd)
-        
+
         self.log.debug(f"Command output: {output}")
-            
+
         if not output or error:
             self.log.error(f"Command failed with error: {error}")
-            
+
         return output, error
 
     def generate_docs(self, num_docs, bucket_name, size, cluster_host=None):
@@ -272,15 +272,15 @@ def generate_docs(self, num_docs, bucket_name, size, cluster_host=None):
             self.username, self.password, size)
 
         cmd += self.cli_flags
-        
+
         self.log.debug(f"Executing command: {cmd}")
         output, error = self._execute_cmd(cmd)
-        
+
         self.log.debug(f"Command output: {output}")
-            
+
         if not output or error:
             self.log.error(f"Command failed with error: {error}")
-            
+
         return output, error
 
     def merge(self, archive_dir, repo_name, start, end):
@@ -297,9 +297,34 @@ def merge(self, archive_dir, repo_name, start, end):
 
         self.log.debug(f"Executing command: {cmd}")
         output, error = self._execute_cmd(cmd)
-        
+
         self.log.debug(f"Command output: {output}")
-            
+
+        if not output or error:
+            self.log.error(f"Command failed with error: {error}")
+
+        return output, error
+
+    def collect_logs(self, archive_dir=None, output_dir=None):
+        """
+        Execute cbbackupmgr collect-logs command to collect logs.
+
+        :param archive_dir str: The location of the backup archive directory.
+        :param output_dir str: The directory to output the collected logs to.
+        """
+        cmd = "%s collect-logs" % self.cbstatCmd
+
+        if archive_dir:
+            cmd += " --archive %s" % archive_dir
+
+        if output_dir:
+            cmd += " --output-dir %s" % output_dir
+
+        self.log.debug(f"Executing command: {cmd}")
+        output, error = self._execute_cmd(cmd)
+
+        self.log.debug(f"Command output: {output}")
+
         if not output or error:
             self.log.error(f"Command failed with error: {error}")
 

couchbase_utils/cb_tools/cbcontbk.py

@@ -19,17 +19,17 @@ def get_cluster_timestamp(self):
         Gets the current UTC timestamp from the cluster host.
         """
         cmd = "date -u +'%Y-%m-%dT%H:%M:%SZ'"
-        
+
         self.log.debug(f"Executing command: {cmd}")
-            
+
         output, error = self._execute_cmd(cmd)
-        
+
         if error:
             self.log.error(f"Failed to get cluster timestamp: {error}")
             return None
-            
+
         self.log.debug(f"Command output: {output}")
-            
+
         return output[0].strip()
 
     def restore(self, archive_path, repo_name,
@@ -66,16 +66,16 @@ def restore(self, archive_path, repo_name,
             cmd += f" --map-data {map_data}"
 
         cmd += self.cli_flags
-        
+
         self.log.debug(f"Executing command: {cmd}")
-            
+
         output, error = self._execute_cmd(cmd)
-        
+
         self.log.debug(f"Command output: {output}")
-            
+
         if not output or error:
             self.log.error(f"Continuous backup restore failed with: {error}")
-            
+
         return output, error
 
     def collect_logs(self, location, temp_dir):
@@ -88,13 +88,13 @@ def collect_logs(self, location, temp_dir):
                f"-d {temp_dir}")
 
         cmd += self.cli_flags
-        
+
         self.log.debug(f"Executing command: {cmd}")
-            
+
         output, error = self._execute_cmd(cmd)
-        
+
         self.log.debug(f"Command output: {output}")
-            
+
         if not output or error:
             self.log.error(f"Command failed with error: {error}")
 

pytests/bucket_collections/collections_base.py

@@ -24,6 +24,7 @@
 from sdk_exceptions import SDKException
 from shell_util.remote_connection import RemoteMachineShellConnection
 from storage.fusion.fusion_base import FusionBase
+from TestInput import TestInputSingleton
 from pytests.bucket_collections.collection_scope_number_manager import CollectionScopeNumberManager
 
 from couchbase.exceptions import InvalidIndexException, \
@@ -230,25 +231,70 @@ def tearDown(self):
 
         # Clean up continuous backup folder
         if self.cont_bkp_test == "NFS":
-            shell = RemoteMachineShellConnection(self.cluster.master)
-            try:
-                self.log.info("Removing continuous backup folder: %s" % self.continuous_backup_location)
-                output, error = shell.execute_command(f"rm -rf {self.continuous_backup_location}")
-                if error:
-                    self.log.warning("Error removing continuous backup folder: %s" % error)
-            except Exception as e:
-                self.log.warning("Exception during cleanup: %s" % str(e))
 
-            # Clean up backup folders
-            try:
-                if hasattr(self, 'repo_name') and self.repo_name:
-                    self.log.info("Removing backup repository")
-                    shell.execute_command(f"rm -rf {self.backup_archive_dir}/{self.backup_repo_name}")
-            except Exception as e:
-                self.log.warning(f"Exception during cleanup: {e}")
+            if self.is_test_failed():
+                self.log.warning("Test failed, skipping cleanup of continuous backup folder to preserve data for investigation: %s" % self.continuous_backup_location)
+                if self.get_cbcollect_info:
+                    self._collect_backup_logs_on_failure()
+            else:
+                shell = RemoteMachineShellConnection(self.cluster.master)
+                try:
+                    self.log.info("Removing continuous backup folder: %s" % self.continuous_backup_location)
+                    output, error = shell.execute_command(f"rm -rf {self.continuous_backup_location}")
+                    if error:
+                        self.log.warning("Error removing continuous backup folder: %s" % error)
+                except Exception as e:
+                    self.log.warning("Exception during cleanup: %s" % str(e))
+
+                # Clean up backup folders
+                try:
+                    if hasattr(self, 'repo_name') and self.repo_name:
+                        self.log.info("Removing backup repository")
+                        shell.execute_command(f"rm -rf {self.backup_archive_dir}/{self.backup_repo_name}")
+                except Exception as e:
+                    self.log.warning(f"Exception during cleanup: {e}")
 
         super(CollectionBase, self).tearDown()
 
+    def _collect_backup_logs_on_failure(self):
+        """
+        Collects cbbackupmgr and cbcontbk logs on test failure.
+        Only runs on Linux nodes. Logs are collected to /data/tmp on the remote
+        node and then copied to the local log path.
+        """
+        log_path = TestInputSingleton.input.param("logs_folder", "/tmp")
+        remote_tmp_dir = "/data/tmp"
+
+        collectors = [
+            ("cbbackupmgr", self.backup_mgr,
+             lambda mgr, tmp: mgr.collect_logs(archive_dir=self.backup_archive_dir,
+                                               output_dir=tmp)),
+            ("cbcontbk", self.cont_bk_mgr,
+             lambda mgr, tmp: mgr.collect_logs(location=self.continuous_backup_location,
+                                               temp_dir=tmp)),
+        ]
+
+        for name, mgr, collect_fn in collectors:
+            try:
+                shell = mgr.shellConn
+                os_info = shell.extract_remote_info()
+                if os_info.type.lower() != "linux":
+                    self.log.info(f"Skipping {name} log collection: OS is not Linux")
+                    continue
+
+                self.log.info(f"Collecting {name} logs for investigation")
+                shell.execute_command(f"mkdir -p {remote_tmp_dir}")
+                collect_fn(mgr, remote_tmp_dir)
+
+                output, _ = shell.execute_command(f"ls {remote_tmp_dir}/*.zip 2>/dev/null")
+                for log_file in output:
+                    log_file = log_file.strip()
+                    if log_file:
+                        self.log.info(f"Copying {log_file} to {log_path}")
+                        shell.get_file(remote_tmp_dir, log_file.split("/")[-1], log_path)
+            except Exception as e:
+                self.log.error(f"Exception during {name} log collection: {e}")
+
     def collection_setup(self):
         ttl_buckets = [
             "multi_bucket.buckets_for_rebalance_tests_with_ttl",