Merge pull request #544 from xcp-ng/stormi/ordering_constraints

Test execution ordering: regroup by axis/package to reduce fixture churn

Author: stormi

conftest.py

@@ -8,6 +8,7 @@
 import logging
 import os
 import tempfile
+from collections import defaultdict
 
 import git
 from cryptography.hazmat.primitives.serialization import SSHCertPrivateKeyTypes
@@ -43,7 +44,7 @@
 # need to import them in the global conftest.py so that they are recognized as fixtures.
 from pkgfixtures import formatted_and_mounted_ext4_disk, sr_disk_wiped
 
-from typing import Any, Dict, Generator, Iterable
+from typing import Any, Dict, Generator, Iterable, List, Optional
 
 # Do we cache VMs?
 try:
@@ -166,10 +167,33 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
             image_format = ["vhd"] # Not giving image-format will default to doing tests on vhd
         metafunc.parametrize("image_format", image_format, scope="session")
 
-def pytest_collection_modifyitems(items: list[pytest.Item], config: pytest.Config) -> None:
-    # Automatically mark tests based on fixtures they require.
-    # Check pytest.ini or pytest --markers for marker descriptions.
 
+# Used to group tests together whenever possible and limit parametrized fixture
+# "context switching" (needless teardown and setup of SRs, for example)
+SCHEDULING_AXES: List[str] = [
+    "image_format",
+]
+
+def get_axis(item: pytest.Item) -> Optional[str]:
+    callspec = getattr(item, "callspec", None)
+    if callspec is None:
+        return None
+    value = None
+    # To keep the ordering simple, the current assumption is that only one axis can be defined for a given test.
+    # For reference, at the time of writing this, the only axis is image_format (qcow2 vs vhd),
+    # attributed to any test requiring the image_format parametrized fixture.
+    for axis in SCHEDULING_AXES:
+        if axis in callspec.params:
+            assert value is None, "we support at most one parametrized axis per test at the moment"
+            value = callspec.getparam(axis)
+    return value
+
+def pytest_collection_modifyitems(items: list[pytest.Item], config: pytest.Config) -> None:
+    """
+    - Automatically mark tests based on fixtures they require.
+      Check pytest.ini or pytest --markers for marker descriptions.
+    - Regroup tests by axis (image_format) and by package (leaf directory)
+    """
     markable_fixtures = [
         'uefi_vm',
         'unix_vm',
@@ -180,6 +204,9 @@ def pytest_collection_modifyitems(items: list[pytest.Item], config: pytest.Confi
         'unused_4k_disks',
     ]
 
+    # -------------
+    # Apply markers
+    # -------------
     for item in items:
         fixturenames = getattr(item, 'fixturenames', ())
         for fixturename in markable_fixtures:
@@ -193,6 +220,48 @@ def pytest_collection_modifyitems(items: list[pytest.Item], config: pytest.Confi
             # multi_vms implies small_vm
             item.add_marker('small_vm')
 
+    # -----------------------------------------------------
+    # Build execution matrix: axis -> leaf package -> items
+    # -----------------------------------------------------
+    axis_ordering: Dict[Optional[str], int] = defaultdict(int)
+    # "None" gets the same order value as the first real axis, on purpose,
+    # so that we may better retain initial test order.
+    # For example, if we start with this test order:
+    # 1. test_A: no axis
+    # 2. test_B: image_format axis, giving us both test_B[vhd] and testB[qcow2]
+    # 3. test_C: no axis
+    # 4. test_D: image_format axis, giving us both test_D[vhd] and testD[qcow2]
+    # We don't want all axis-less tests grouped at the beginning:
+    #     test_A -> test_C -> test_B[vhd ] -> test_D[vhd] -> test_B[qcow2] -> test_D[qcow2]
+    # Instead, we want to keep the intial order as much as possible:
+    #     test_A -> test_B[vhd ] -> test_C -> test_D[vhd] -> test_B[qcow2] -> test_D[qcow2]
+    # Here only the two QCOW2 tests get pushed to the back in order to limit context switching from VHD to QCOW2.
+    axis_ordering[None] = 1
+
+    for item in items:
+        axis = get_axis(item)
+        if axis not in axis_ordering:
+            axis_ordering[axis] = len(axis_ordering) # 1 (same as None's order), then 2, etc.
+
+    grouped: dict[int, dict[pytest.Package, list[pytest.Item]]] = defaultdict(lambda: defaultdict(list))
+
+    # List the items in the order that pytest initially determined, and add extra grouping criteria.
+    for item in items:
+        axis_order = axis_ordering[get_axis(item)]
+        package = item.getparent(pytest.Package)
+        assert package is not None, "all items must come from a package"
+        grouped[axis_order][package].append(item)
+
+    # Flatten back to a list of items
+    new_items: List[pytest.Item] = [
+        item
+        for axis_order in sorted(grouped) # apply axis_ordering here
+        for package in grouped[axis_order]
+        for item in grouped[axis_order][package]
+    ]
+    items[:] = new_items
+
+
 # BEGIN make test results visible from fixtures
 # from https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures