diff --git a/virttest/env_process.py b/virttest/env_process.py index 5f622fd71a..5e95e6bc8f 100644 --- a/virttest/env_process.py +++ b/virttest/env_process.py @@ -16,7 +16,7 @@ from avocado.core import exceptions from avocado.utils import archive from avocado.utils import cpu as cpu_utils -from avocado.utils import crypto +from avocado.utils import crypto, path from avocado.utils import process as a_process from six.moves import xrange @@ -43,7 +43,11 @@ from virttest._wrappers import lazy_import from virttest.test_setup.aexpect import KillTailThreads from virttest.test_setup.core import SetupManager -from virttest.test_setup.gcov import ResetQemuGCov +from virttest.test_setup.gcov import ( + ResetGCov, + collect_gcovr_coverage, + collect_lcov_coverage, +) from virttest.test_setup.kernel import KSMSetup, ReloadKVMModules from virttest.test_setup.libvirt_setup import LibvirtdDebugLogConfig from virttest.test_setup.memory import HugePagesSetup, TransparentHugePagesSetup @@ -1000,7 +1004,7 @@ def preprocess(test, params, env): # and last running during pre/postprocess. That way vms will be actually # off to ensure data is written to disk. _setup_manager.register(ProcessVMOff) - _setup_manager.register(ResetQemuGCov) + _setup_manager.register(ResetGCov) _setup_manager.register(VerifyHostDMesg) _setup_manager.register(SwitchSMTOff) _setup_manager.register(CheckRunningAsRoot) @@ -1319,32 +1323,84 @@ def postprocess(test, params, env): # Collect code coverage report for qemu if enabled if params.get("gcov_qemu", "no") == "yes": - qemu_builddir = os.path.join(test.bindir, "build", "qemu") - if os.path.isdir(qemu_builddir) and utils_package.package_install("gcovr"): - gcov_qemu_dir = utils_misc.get_path(test.debugdir, "gcov_qemu") - os.makedirs(gcov_qemu_dir) - os.chdir(qemu_builddir) - collect_cmd_opts = params.get("gcov_qemu_collect_cmd_opts", "--html") - online_count = ( - cpu_utils.online_count() - if hasattr(cpu_utils, "online_count") - else cpu_utils.online_cpus_count() - ) - collect_cmd = "gcovr -j %s -o %s -s %s ." % ( - online_count, - os.path.join(gcov_qemu_dir, "gcov.html"), - collect_cmd_opts, - ) - a_process.system(collect_cmd, shell=True) - if params.get("gcov_qemu_compress", "no") == "yes": - os.chdir(test.debugdir) - archive.compress("gcov_qemu.tar.gz", gcov_qemu_dir) - shutil.rmtree(gcov_qemu_dir, ignore_errors=True) + qemu_builddir = params.get( + "gcov_qemu_builddir", os.path.join(test.bindir, "build", "qemu") + ) + gcov_format = params.get("gcov_qemu_format", "html") + test_name = params.get("shortname", getattr(test, "name", "unknown_test")) + if hasattr(test_name, "uid"): + test_name = str(test_name.uid) + + if gcov_format == "lcov": + try: + path.find_command("lcov") + except path.CmdNotFoundError: + LOG.warning("lcov package not installed, cannot collect QEMU coverage") + else: + gcov_qemu_dir = utils_misc.get_path(test.debugdir, "gcov_qemu") + collect_lcov_coverage(qemu_builddir, gcov_qemu_dir, test_name, "qemu") + + if params.get("gcov_qemu_compress", "no") == "yes": + os.chdir(test.debugdir) + archive.compress("gcov_qemu.tar.gz", gcov_qemu_dir) + shutil.rmtree(gcov_qemu_dir, ignore_errors=True) else: - LOG.warning( - "Check either qemu build directory availablilty" - " or install gcovr package for qemu coverage report" - ) + if utils_package.package_install("gcovr"): + gcov_qemu_dir = utils_misc.get_path(test.debugdir, "gcov_qemu") + collect_cmd_opts = params.get("gcov_qemu_collect_cmd_opts", "--html") + collect_gcovr_coverage( + qemu_builddir, gcov_qemu_dir, "qemu", collect_cmd_opts + ) + + if params.get("gcov_qemu_compress", "no") == "yes": + os.chdir(test.debugdir) + archive.compress("gcov_qemu.tar.gz", gcov_qemu_dir) + shutil.rmtree(gcov_qemu_dir, ignore_errors=True) + else: + LOG.warning("gcovr package not installed, cannot collect QEMU coverage") + + # Collect code coverage report for libvirt if enabled + if params.get("gcov_libvirt", "no") == "yes": + libvirt_builddir = params.get("gcov_libvirt_builddir", "/var/tmp/libvirt") + gcov_format = params.get("gcov_libvirt_format", "html") + test_name = params.get("shortname", getattr(test, "name", "unknown_test")) + if hasattr(test_name, "uid"): + test_name = str(test_name.uid) + + if gcov_format == "lcov": + try: + path.find_command("lcov") + except path.CmdNotFoundError: + LOG.warning( + "lcov package not installed, cannot collect libvirt coverage" + ) + else: + gcov_libvirt_dir = utils_misc.get_path(test.debugdir, "gcov_libvirt") + collect_lcov_coverage( + libvirt_builddir, gcov_libvirt_dir, test_name, "libvirt" + ) + + if params.get("gcov_libvirt_compress", "no") == "yes": + os.chdir(test.debugdir) + archive.compress("gcov_libvirt.tar.gz", gcov_libvirt_dir) + shutil.rmtree(gcov_libvirt_dir, ignore_errors=True) + else: + if utils_package.package_install("gcovr"): + gcov_libvirt_dir = utils_misc.get_path(test.debugdir, "gcov_libvirt") + collect_cmd_opts = params.get("gcov_libvirt_collect_cmd_opts", "--html") + collect_gcovr_coverage( + libvirt_builddir, gcov_libvirt_dir, "libvirt", collect_cmd_opts + ) + + if params.get("gcov_libvirt_compress", "no") == "yes": + os.chdir(test.debugdir) + archive.compress("gcov_libvirt.tar.gz", gcov_libvirt_dir) + shutil.rmtree(gcov_libvirt_dir, ignore_errors=True) + else: + LOG.warning( + "gcovr package not installed, cannot collect libvirt coverage" + ) + # Postprocess all VMs and images try: process( diff --git a/virttest/shared/cfg/base.cfg b/virttest/shared/cfg/base.cfg index 10241c7700..5b7618d54d 100644 --- a/virttest/shared/cfg/base.cfg +++ b/virttest/shared/cfg/base.cfg @@ -904,6 +904,7 @@ sysprep_options = "--operations machine-id" # Enable Code Coverage report # prerequisite: qemu build test is run with --enable-gcov +# or libvirt is built with --enable-coverage # configure option and preserve_srcdir = yes # and qemu build dir is available # Enable/disable gcov for qemu, default: disable @@ -911,6 +912,10 @@ gcov_qemu = no # Enable/disable to reset the code coverage report # generated/collected by previous test gcov_qemu_reset = yes +# Coverage output format: lcov (generates .info tracefiles) or html (generates HTML reports) +# For per-test lcov tracefiles with --test-name, use "lcov" +# For aggregated HTML reports with gcovr, use "html" +gcov_qemu_format = html # Additional command options for gcovr # E:g:- "--html-details --html --exclude-directories=capstone" # above options will be required to collect detailed html @@ -920,6 +925,32 @@ gcov_qemu_reset = yes gcov_qemu_collect_cmd_opts = "--html" # Enable/disable to compress code coverage report gcov_qemu_compress = no +# Qemu build directory where .gcda files are generated +gcov_qemu_builddir = /var/tmp/qemu +# Fail test if coverage reset fails +gcov_qemu_reset_strict = yes + +# Libvirt code coverage configuration +# Enable/disable gcov for libvirt, default: disable +gcov_libvirt = no +# Enable/disable to reset the libvirt coverage report +# generated/collected by previous test +gcov_libvirt_reset = yes +# Coverage output format: lcov or html +gcov_libvirt_format = lcov +# Libvirt build directory where .gcda files are generated +gcov_libvirt_builddir = /var/tmp/libvirt +# Additional command options for gcovr when format is html +# E.g.: "--html-details --html --exclude-unreachable-branches" +gcov_libvirt_collect_cmd_opts = "--html" +# Enable/disable to compress libvirt code coverage report +gcov_libvirt_compress = no +# Libvirt daemon to restart after coverage reset (if needed) +gcov_libvirt_daemon = libvirtd +# Enable/disable restarting libvirt daemon after coverage reset +gcov_libvirt_restart_daemon = no +# Fail test if coverage reset fails +gcov_libvirt_reset_strict = yes Linux: # param for installing stress tool from repo diff --git a/virttest/test_setup/gcov.py b/virttest/test_setup/gcov.py index fa037b72bb..a1021f3a60 100644 --- a/virttest/test_setup/gcov.py +++ b/virttest/test_setup/gcov.py @@ -1,28 +1,261 @@ +import logging import os from avocado.utils import process as a_process +from avocado.utils import service from virttest.test_setup.core import Setuper +LOG = logging.getLogger("avocado." + __name__) + + +def collect_lcov_coverage(build_dir, output_dir, test_name, component): + """ + Collect coverage data using lcov with per-test naming. + + :param build_dir: Build directory containing .gcda files + :param output_dir: Output directory for coverage files + :param test_name: Name of the test for tracefile naming + :param component: Component name (qemu or libvirt) + :return: Path to generated tracefile, or None on failure + """ + if not os.path.isdir(build_dir): + LOG.warning("%s build directory not found: %s", component, build_dir) + return None + + os.makedirs(output_dir, exist_ok=True) + tracefile = os.path.join(output_dir, "coverage_%s.info" % test_name) + + # Collect coverage data with test name + collect_cmd = ( + "lcov --capture " + "--directory %s " + "--output-file %s " + "--test-name %s " % (build_dir, tracefile, test_name) + ) + + try: + a_process.system(collect_cmd, shell=True) + + # Validate the generated file + if os.path.exists(tracefile): + file_size = os.path.getsize(tracefile) + if file_size > 0: + LOG.info( + "%s coverage tracefile saved: %s (%d bytes)", + component.upper(), + tracefile, + file_size, + ) + return tracefile + else: + LOG.warning( + "%s coverage file is empty, removing: %s", component, tracefile + ) + os.unlink(tracefile) + return None + else: + LOG.warning("%s coverage file was not created: %s", component, tracefile) + return None + + except Exception as e: + LOG.error("Failed to collect %s coverage: %s", component, e) + return None + + +def collect_gcovr_coverage(build_dir, output_dir, component, cmd_opts="--html"): + """ + Collect coverage data using gcovr in the specified format. + + :param build_dir: Build directory containing .gcda files + :param output_dir: Output directory for coverage report + :param component: Component name (qemu or libvirt) + :param cmd_opts: Additional gcovr command options + (e.g., "--html", "--xml", "--json", "--csv", "--txt") + :return: Path to generated coverage file, or None on failure + """ + if not os.path.isdir(build_dir): + LOG.warning("%s build directory not found: %s", component, build_dir) + return None + + try: + # Import here to avoid circular dependencies + from avocado.utils import cpu as cpu_utils + + os.makedirs(output_dir, exist_ok=True) + + # Map gcovr format flags to output filenames and display names + format_map = { + "--txt": ("gcov.txt", "TXT"), + "--xml": ("gcov.xml", "XML"), + "--json": ("gcov.json", "JSON"), + "--markdown": ("gcov.md", "MARKDOWN"), + "--csv": ("gcov.csv", "CSV"), + "--clover": ("gcov-clover.xml", "CLOVER"), + "--cobertura": ("gcov-cobertura.xml", "COBERTURA"), + "--lcov": ("gcov.lcov", "LCOV"), + "--sonarqube": ("gcov-sonarqube.xml", "SonarQube XML"), + "--coveralls": ("gcov-coveralls.json", "Coveralls JSON"), + } + + output_filename = "gcov.html" + format_name = "HTML" + for flag, (filename, name) in format_map.items(): + if flag in cmd_opts: + output_filename = filename + format_name = name + break + + output_file = os.path.join(output_dir, output_filename) + + # Get CPU count for parallel processing + online_count = ( + cpu_utils.online_count() + if hasattr(cpu_utils, "online_count") + else cpu_utils.online_cpus_count() + ) + + # Change to build directory for gcovr + original_dir = os.getcwd() + os.chdir(build_dir) + + # Build gcovr command + collect_cmd = "gcovr -j %s -o %s -s %s ." % ( + online_count, + output_file, + cmd_opts, + ) + + LOG.info("Collecting %s %s coverage report...", component.upper(), format_name) + a_process.system(collect_cmd, shell=True) + + # Restore original directory + os.chdir(original_dir) + + # Validate the generated file + if os.path.exists(output_file): + file_size = os.path.getsize(output_file) + if file_size > 0: + LOG.info( + "%s coverage %s report saved: %s (%d bytes)", + component.upper(), + format_name, + output_file, + file_size, + ) + return output_file + else: + LOG.warning( + "%s %s report is empty: %s", component, format_name, output_file + ) + return None + else: + LOG.warning( + "%s %s report was not created: %s", component, format_name, output_file + ) + return None + + except Exception as e: + LOG.error("Failed to collect %s gcovr coverage: %s", component, e) + # Restore directory on error + try: + os.chdir(original_dir) + except Exception: + pass + return None + + +class ResetGCov(Setuper): + """Reset code coverage data for QEMU and/or libvirt before each test.""" -class ResetQemuGCov(Setuper): def setup(self): # Check if code coverage for qemu is enabled and # if coverage reset is enabled too, reset coverage report gcov_qemu = self.params.get("gcov_qemu", "no") == "yes" gcov_qemu_reset = self.params.get("gcov_qemu_reset", "no") == "yes" if gcov_qemu and gcov_qemu_reset: - qemu_builddir = os.path.join(self.test.bindir, "build", "qemu") + qemu_builddir = self.params.get( + "gcov_qemu_builddir", os.path.join(self.test.bindir, "build", "qemu") + ) qemu_bin = os.path.join(self.test.bindir, "bin", "qemu") if os.path.isdir(qemu_builddir) and os.path.isfile(qemu_bin): os.chdir(qemu_builddir) - # Looks like libvirt process does not have permissions to write to - # coverage files, hence give write for all files in qemu source + LOG.info("Resetting QEMU code coverage data") + # Give write permissions for coverage files + # (libvirt/qemu process may need write access) reset_cmd = "make clean-coverage;%s -version;" % qemu_bin reset_cmd += ( 'find %s -name "*.gcda" -exec chmod a=rwx {} \;' % qemu_builddir ) - a_process.system(reset_cmd, shell=True) + try: + a_process.system(reset_cmd, shell=True) + LOG.info("QEMU coverage data reset successfully") + except Exception as e: + LOG.warning("Failed to reset QEMU coverage: %s", e) + strict_reset = ( + self.params.get("gcov_qemu_reset_strict", "yes") == "yes" + ) + if strict_reset: + raise e + + # Check if code coverage for libvirt is enabled + gcov_libvirt = self.params.get("gcov_libvirt", "no") == "yes" + gcov_libvirt_reset = self.params.get("gcov_libvirt_reset", "no") == "yes" + if gcov_libvirt and gcov_libvirt_reset: + libvirt_builddir = self.params.get( + "gcov_libvirt_builddir", "/var/tmp/libvirt" + ) + if os.path.isdir(libvirt_builddir): + LOG.info("Resetting libvirt code coverage data in %s", libvirt_builddir) + + # Find and remove .gcda files + reset_cmd = 'find %s -name "*.gcda" -type f -delete' % libvirt_builddir + + # Also reset lcov counters if lcov is available + lcov_reset = ( + "lcov --zerocounters --directory %s 2>/dev/null || true" + % libvirt_builddir + ) + + # Fix permissions for coverage files + chmod_cmd = ( + 'find %s -name "*.gcda" -o -name "*.gcno" | xargs chmod a+rw 2>/dev/null || true' + % libvirt_builddir + ) + + full_reset_cmd = "%s; %s; %s" % (reset_cmd, lcov_reset, chmod_cmd) + + try: + a_process.system(full_reset_cmd, shell=True) + LOG.info("Libvirt coverage data reset successfully") + + # Optionally restart libvirt daemon to ensure clean state + restart_daemon = ( + self.params.get("gcov_libvirt_restart_daemon", "no") == "yes" + ) + if restart_daemon: + daemon_name = self.params.get( + "gcov_libvirt_daemon", "virtqemud" + ) + try: + libvirt_service = service.Factory.create_service( + daemon_name + ) + libvirt_service.restart() + LOG.info( + "Restarted %s daemon for clean coverage state", + daemon_name, + ) + except Exception as e: + LOG.warning("Failed to restart %s: %s", daemon_name, e) + + except Exception as e: + LOG.warning("Failed to reset libvirt coverage: %s", e) + strict_reset = ( + self.params.get("gcov_libvirt_reset_strict", "yes") == "yes" + ) + if strict_reset: + raise e def cleanup(self): pass