Merge pull request #4285 from dzhengfy/return_qemu_log_migration

migration: Retrieve QEMU logs after migration failure

Author: Yingshun

virttest/migration.py

@@ -18,6 +18,7 @@
     utils_iptables,
     utils_misc,
     utils_net,
+    utils_sys,
     utils_test,
 )
 
@@ -372,6 +373,7 @@ def do_migration(
                                     {"func": <function domjobabort at 0x7f5835cd08c8>,
                                      "before_pause": "no"}
                                    ]
+        :param virsh_opt: str, virsh options
         :param args: dictionary used by func,
                      'func_param' is mandatory for func parameter.
                      If no real func_param, none is requested.
@@ -709,7 +711,7 @@ def _do_orderly_migration(
             raise exceptions.TestFail()
 
         LOG.info("Checking migration result...")
-        self.check_result(self.ret, args)
+        self.check_result(self.ret, args, vms)
 
     def cleanup_dest_vm(self, vm, srcuri, desturi):
         """
@@ -830,17 +832,19 @@ def update_virsh_migrate_extra_args(self, params):
         extra_args.update({"err_msg": params.get("err_msg")})
         return extra_args
 
-    def check_result(self, result, params):
+    def check_result(self, result, params, vms):
         """
         Check if the migration result is as expected
 
         :param result: the output of migration
         :param params: the parameters dict
-        :raise: test.fail if test is failed
+        :param vms: vm instances
+        :raise: exceptions.TestFail if test is failed
         """
         status_error = "yes" == params.get("status_error", "no")
         err_msg = params.get("err_msg")
         if not result:
+            utils_sys.get_qemu_log(vms, type="both", params=params)
             raise exceptions.TestError("No migration result is returned.")
 
         LOG.info("Migration out: %s", result.stdout_text.strip())
@@ -849,6 +853,7 @@ def check_result(self, result, params):
         if status_error:  # Migration should fail
             if err_msg:  # Special error messages are expected
                 if not re.search(err_msg, result.stderr_text.strip()):
+                    utils_sys.get_qemu_log(vms, type="both", params=params)
                     raise exceptions.TestFail(
                         "Can not find the expected "
                         "patterns '%s' in output '%s'"
@@ -863,6 +868,7 @@ def check_result(self, result, params):
                     raise exceptions.TestFail("Migration success is unexpected result")
         else:
             if int(result.exit_status) != 0:
+                utils_sys.get_qemu_log(vms, type="both", params=params)
                 raise exceptions.TestFail(result.stderr_text.strip())
 
     def do_cancel(self, sig=signal.SIGKILL):

virttest/migration_template.py

@@ -510,7 +510,7 @@ def _migrate(self):
         )
 
         LOG.info("Check migration result: succeed or fail with expected error")
-        self.obj_migration.check_result(self.obj_migration.ret, self.params)
+        self.obj_migration.check_result(self.obj_migration.ret, self.params, self.vms)
 
         # Check "suspended post-copy" event after postcopy migration
         if self.migrate_flags & VIR_MIGRATE_POSTCOPY:
@@ -563,7 +563,7 @@ def _migrate_back(self):
         )
 
         LOG.info("Check migration result: succeed or fail with expected error")
-        self.obj_migration.check_result(self.obj_migration.ret, self.params)
+        self.obj_migration.check_result(self.obj_migration.ret, self.params, self.vms)
 
         # Set vm connect_uri to self.src_uri if migration back succeeds
         if self.obj_migration.ret.exit_status == 0:

virttest/utils_sys.py

@@ -9,6 +9,7 @@
 
 from avocado.utils import process
 
+from virttest import remote
 from virttest.utils_misc import cmd_status_output
 from virttest.utils_test import libvirt
 
@@ -133,3 +134,56 @@ def get_pids_for(process_names, sort_pids=True, session=None):
         relevant_pids.sort()
 
     return relevant_pids
+
+
+def get_qemu_log(vms, type="local", params=None, log_lines=10):
+    """
+    Get last N lines of QEMU log from local host and remote host.
+
+    :param vms: VM objects
+    :param type: str, valid values: "local", "remote" or "both"
+    :param params: dict, test parameters
+    :param log_lines: int, number of last lines to retrieve from log, default 10
+    :return: list, like [{"vm_name": "vm1", "local": xxx, "remote": xxx}, {"vm_name": "vm2", "local": xxx}]
+    """
+    logs = []
+    if params is not None and type != "local":
+        server_ip = params.get("migrate_dest_host", params.get("remote_ip"))
+        server_user = params.get("server_user", params.get("remote_user"))
+        server_pwd = params.get("server_pwd", params.get("remote_pwd"))
+        if server_ip:
+            server_session = remote.wait_for_login(
+                "ssh", server_ip, "22", server_user, server_pwd, r"[\#\$]\s*$"
+            )
+        else:
+            LOG.warning("Remote host IP not found in params")
+    try:
+        for vm in vms:
+            log_contents = {"vm_name": vm.name}
+            log_file = "/var/log/libvirt/qemu/%s.log" % vm.name
+            cmd = "tail -n %d %s 2>/dev/null || echo 'Log file not found'" % (
+                log_lines,
+                log_file,
+            )
+            if type in ["local", "both"]:
+                result = process.run(cmd, shell=True, ignore_status=True, verbose=False)
+                log_content = result.stdout_text.strip()
+                LOG.debug(
+                    "QEMU log from source host for vm %s:\n%s", vm.name, log_content
+                )
+                log_contents.update({"local": log_content})
+
+            if type in ["remote", "both"] and server_session is not None:
+                log_content = server_session.cmd_output(cmd, timeout=10).strip()
+                LOG.debug(
+                    "QEMU log from remote host for vm %s:\n%s", vm.name, log_content
+                )
+                log_contents.update({"remote": log_content})
+            logs.append(log_contents)
+
+    except Exception as detail:
+        LOG.warning("Failed to get QEMU log: %s", detail)
+    finally:
+        if server_session:
+            server_session.close()
+    return logs