Allow for configurable extra vm dump checks for present/absent messages

The typical use for this is when dumping extra logs and wanting to
make sure certain information is always present (e.g. startup, etc.)
and certain information is always absent (e.g. segfaults, etc.).

It makes use of aexpect's linux ops as well as pattern matching
for grepping the expected or unexpected messages.

Signed-off-by: Plamen Dimitrov <plamen.dimitrov@intra2net.com>

Author: pevogam

virttest/env_process.py

@@ -12,6 +12,7 @@
 import time
 
 import six
+from aexpect import ops_linux as ops
 from aexpect import remote
 from avocado.core import exceptions
 from avocado.utils import archive
@@ -667,6 +668,60 @@ def postprocess_vm(test, params, env, name):
                     serial_login=serial_login,
                 )
 
+    if params.get("vm_extra_dump_paths") is not None:
+        # even if there is previously existing session we cannot guarantee its state
+        # is viable, e.g. it may be in python .env or SMTP sub-session, etc.
+        if not vm.is_alive():
+            LOG.warning("VM is not alive so we cannot download extra dump paths")
+        else:
+            session = vm.wait_for_login(timeout=vm.LOGIN_WAIT_TIMEOUT)
+            vm_extra_dumps = os.path.join(test.outputdir, "vm_extra_dumps", vm.name)
+            if not os.path.exists(vm_extra_dumps):
+                os.makedirs(vm_extra_dumps)
+            for i, dump_path in enumerate(
+                params.get_list("vm_extra_dump_paths", delimiter=";")
+            ):
+                errors = []
+                if params.get_boolean(f"verify_vm_extra_dump_{i}"):
+                    if vm.params["os_type"] == "windows":
+                        LOG.warning(
+                            "Extra dump file validation is not supported on Windows"
+                        )
+                    LOG.info(f"Checking for expected messages in {dump_path}")
+                    for message in params.get_list(
+                        f"expected_vm_extra_dump_{i}", delimiter=";"
+                    ):
+                        if not ops.grep(
+                            session, message, dump_path, check=True, flags="-aP"
+                        ):
+                            errors.append(
+                                f"Missing expected message '{message}' "
+                                f"in {vm.name} extra dump {dump_path}"
+                            )
+                    for message in params.get_list(
+                        f"unexpected_vm_extra_dump_{i}", delimiter=";"
+                    ):
+                        if ops.grep(
+                            session, message, dump_path, check=True, flags="-aP"
+                        ):
+                            errors.append(
+                                f"Redundant unexpected message '{message}' "
+                                f"in {vm.name} extra dump {dump_path}"
+                            )
+                try:
+                    vm.copy_files_from(dump_path, vm_extra_dumps)
+                except:
+                    LOG.error(
+                        "Could not copy the extra dump '%s' from the vm '%s'",
+                        dump_path,
+                        vm.name,
+                    )
+                if errors:
+                    raise exceptions.TestFail(
+                        f"Errors during validation of log messages dumped "
+                        f"from {vm.name}:\n" + "\n".join(errors)
+                    )
+
     # Close all SSH sessions that might be active to this VM
     for s in vm.remote_sessions[:]:
         try:
@@ -677,20 +732,6 @@ def postprocess_vm(test, params, env, name):
 
     utils_logfile.close_log_file()
 
-    if params.get("vm_extra_dump_paths") is not None:
-        vm_extra_dumps = os.path.join(test.outputdir, "vm_extra_dumps")
-        if not os.path.exists(vm_extra_dumps):
-            os.makedirs(vm_extra_dumps)
-        for dump_path in params.get("vm_extra_dump_paths").split(";"):
-            try:
-                vm.copy_files_from(dump_path, vm_extra_dumps)
-            except:
-                LOG.error(
-                    "Could not copy the extra dump '%s' from the vm '%s'",
-                    dump_path,
-                    vm.name,
-                )
-
     if params.get("kill_vm") == "yes":
         kill_vm_timeout = float(params.get("kill_vm_timeout", 0))
         if kill_vm_timeout: