diff --git a/conftest.py b/conftest.py index deffc6e59..afba14537 100644 --- a/conftest.py +++ b/conftest.py @@ -8,6 +8,7 @@ import logging import os import tempfile +from collections import defaultdict import git from cryptography.hazmat.primitives.serialization import SSHCertPrivateKeyTypes @@ -43,7 +44,7 @@ # need to import them in the global conftest.py so that they are recognized as fixtures. from pkgfixtures import formatted_and_mounted_ext4_disk, sr_disk_wiped -from typing import Any, Dict, Generator, Iterable +from typing import Any, Dict, Generator, Iterable, List, Optional # Do we cache VMs? try: @@ -166,10 +167,33 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: image_format = ["vhd"] # Not giving image-format will default to doing tests on vhd metafunc.parametrize("image_format", image_format, scope="session") -def pytest_collection_modifyitems(items: list[pytest.Item], config: pytest.Config) -> None: - # Automatically mark tests based on fixtures they require. - # Check pytest.ini or pytest --markers for marker descriptions. +# Used to group tests together whenever possible and limit parametrized fixture +# "context switching" (needless teardown and setup of SRs, for example) +SCHEDULING_AXES: List[str] = [ + "image_format", +] + +def get_axis(item: pytest.Item) -> Optional[str]: + callspec = getattr(item, "callspec", None) + if callspec is None: + return None + value = None + # To keep the ordering simple, the current assumption is that only one axis can be defined for a given test. + # For reference, at the time of writing this, the only axis is image_format (qcow2 vs vhd), + # attributed to any test requiring the image_format parametrized fixture. + for axis in SCHEDULING_AXES: + if axis in callspec.params: + assert value is None, "we support at most one parametrized axis per test at the moment" + value = callspec.getparam(axis) + return value + +def pytest_collection_modifyitems(items: list[pytest.Item], config: pytest.Config) -> None: + """ + - Automatically mark tests based on fixtures they require. + Check pytest.ini or pytest --markers for marker descriptions. + - Regroup tests by axis (image_format) and by package (leaf directory) + """ markable_fixtures = [ 'uefi_vm', 'unix_vm', @@ -180,6 +204,9 @@ def pytest_collection_modifyitems(items: list[pytest.Item], config: pytest.Confi 'unused_4k_disks', ] + # ------------- + # Apply markers + # ------------- for item in items: fixturenames = getattr(item, 'fixturenames', ()) for fixturename in markable_fixtures: @@ -193,6 +220,48 @@ def pytest_collection_modifyitems(items: list[pytest.Item], config: pytest.Confi # multi_vms implies small_vm item.add_marker('small_vm') + # ----------------------------------------------------- + # Build execution matrix: axis -> leaf package -> items + # ----------------------------------------------------- + axis_ordering: Dict[Optional[str], int] = defaultdict(int) + # "None" gets the same order value as the first real axis, on purpose, + # so that we may better retain initial test order. + # For example, if we start with this test order: + # 1. test_A: no axis + # 2. test_B: image_format axis, giving us both test_B[vhd] and testB[qcow2] + # 3. test_C: no axis + # 4. test_D: image_format axis, giving us both test_D[vhd] and testD[qcow2] + # We don't want all axis-less tests grouped at the beginning: + # test_A -> test_C -> test_B[vhd ] -> test_D[vhd] -> test_B[qcow2] -> test_D[qcow2] + # Instead, we want to keep the intial order as much as possible: + # test_A -> test_B[vhd ] -> test_C -> test_D[vhd] -> test_B[qcow2] -> test_D[qcow2] + # Here only the two QCOW2 tests get pushed to the back in order to limit context switching from VHD to QCOW2. + axis_ordering[None] = 1 + + for item in items: + axis = get_axis(item) + if axis not in axis_ordering: + axis_ordering[axis] = len(axis_ordering) # 1 (same as None's order), then 2, etc. + + grouped: dict[int, dict[pytest.Package, list[pytest.Item]]] = defaultdict(lambda: defaultdict(list)) + + # List the items in the order that pytest initially determined, and add extra grouping criteria. + for item in items: + axis_order = axis_ordering[get_axis(item)] + package = item.getparent(pytest.Package) + assert package is not None, "all items must come from a package" + grouped[axis_order][package].append(item) + + # Flatten back to a list of items + new_items: List[pytest.Item] = [ + item + for axis_order in sorted(grouped) # apply axis_ordering here + for package in grouped[axis_order] + for item in grouped[axis_order][package] + ] + items[:] = new_items + + # BEGIN make test results visible from fixtures # from https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures