Test execution ordering: regroup by axis/package to reduce fixture churn

pytest's default ordering could interleave image_format=vhd/qcow2
tests in a way that repeatedly destroys and recreates SRs.
It could also choose to cross package boundaries as an attempt to
optimize test runs, but we'd prefer to see tests logically grouped,
for easier log analysis and fixture management (which is perfectible
and which we'll have to improve anyway).

We start with pytest's computed ordering, then regroup tests by
parametrized axis (currently only image_format) and by leaf python
package, in order to keep related tests together and reduce needless
costly context switching.

Axis-less tests intentionally share the same ordering level as the
first parametrized axis so they stay near their original position
instead of being grouped at the beginning of the run.

Interfering with pytest's ordering is not something to do lightly, so
it's possible that we may discover issues caused by this. But meanwhile,
this has shown a notable reduction (10%) of the number of setup/teardown
operations when both vhd and qcow2 image formats are set, and test order
closer to what we'd intuitively expect.

Signed-off-by: Samuel Verschelde <stormi-xcp@ylix.fr>

Author: stormi

conftest.py

@@ -8,6 +8,7 @@
 import logging
 import os
 import tempfile
+from collections import defaultdict
 
 import git
 from cryptography.hazmat.primitives.serialization import SSHCertPrivateKeyTypes
@@ -43,7 +44,7 @@
 # need to import them in the global conftest.py so that they are recognized as fixtures.
 from pkgfixtures import formatted_and_mounted_ext4_disk, sr_disk_wiped
 
-from typing import Any, Dict, Generator, Iterable
+from typing import Any, Dict, Generator, Iterable, List, Optional
 
 # Do we cache VMs?
 try:
@@ -166,10 +167,33 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
             image_format = ["vhd"] # Not giving image-format will default to doing tests on vhd
         metafunc.parametrize("image_format", image_format, scope="session")
 
-def pytest_collection_modifyitems(items: list[pytest.Item], config: pytest.Config) -> None:
-    # Automatically mark tests based on fixtures they require.
-    # Check pytest.ini or pytest --markers for marker descriptions.
 
+# Used to group tests together whenever possible and limit parametrized fixture
+# "context switching" (needless teardown and setup of SRs, for example)
+SCHEDULING_AXES: List[str] = [
+    "image_format",
+]
+
+def get_axis(item: pytest.Item) -> Optional[str]:
+    callspec = getattr(item, "callspec", None)
+    if callspec is None:
+        return None
+    value = None
+    # To keep the ordering simple, the current assumption is that only one axis can be defined for a given test.
+    # For reference, at the time of writing this, the only axis is image_format (qcow2 vs vhd),
+    # attributed to any test requiring the image_format parametrized fixture.
+    for axis in SCHEDULING_AXES:
+        if axis in callspec.params:
+            assert value is None, "we support at most one parametrized axis per test at the moment"
+            value = callspec.getparam(axis)
+    return value
+
+def pytest_collection_modifyitems(items: list[pytest.Item], config: pytest.Config) -> None:
+    """
+    - Automatically mark tests based on fixtures they require.
+      Check pytest.ini or pytest --markers for marker descriptions.
+    - Regroup tests by axis (image_format) and by package (leaf directory)
+    """
     markable_fixtures = [
         'uefi_vm',
         'unix_vm',
@@ -180,6 +204,9 @@ def pytest_collection_modifyitems(items: list[pytest.Item], config: pytest.Confi
         'unused_4k_disks',
     ]
 
+    # -------------
+    # Apply markers
+    # -------------
     for item in items:
         fixturenames = getattr(item, 'fixturenames', ())
         for fixturename in markable_fixtures:
@@ -193,6 +220,48 @@ def pytest_collection_modifyitems(items: list[pytest.Item], config: pytest.Confi
             # multi_vms implies small_vm
             item.add_marker('small_vm')
 
+    # -----------------------------------------------------
+    # Build execution matrix: axis -> leaf package -> items
+    # -----------------------------------------------------
+    axis_ordering: Dict[Optional[str], int] = defaultdict(int)
+    # "None" gets the same order value as the first real axis, on purpose,
+    # so that we may better retain initial test order.
+    # For example, if we start with this test order:
+    # 1. test_A: no axis
+    # 2. test_B: image_format axis, giving us both test_B[vhd] and testB[qcow2]
+    # 3. test_C: no axis
+    # 4. test_D: image_format axis, giving us both test_D[vhd] and testD[qcow2]
+    # We don't want all axis-less tests grouped at the beginning:
+    #     test_A -> test_C -> test_B[vhd ] -> test_D[vhd] -> test_B[qcow2] -> test_D[qcow2]
+    # Instead, we want to keep the intial order as much as possible:
+    #     test_A -> test_B[vhd ] -> test_C -> test_D[vhd] -> test_B[qcow2] -> test_D[qcow2]
+    # Here only the two QCOW2 tests get pushed to the back in order to limit context switching from VHD to QCOW2.
+    axis_ordering[None] = 1
+
+    for item in items:
+        axis = get_axis(item)
+        if axis not in axis_ordering:
+            axis_ordering[axis] = len(axis_ordering) # 1 (same as None's order), then 2, etc.
+
+    grouped: dict[int, dict[pytest.Package, list[pytest.Item]]] = defaultdict(lambda: defaultdict(list))
+
+    # List the items in the order that pytest initially determined, and add extra grouping criteria.
+    for item in items:
+        axis_order = axis_ordering[get_axis(item)]
+        package = item.getparent(pytest.Package)
+        assert package is not None, "all items must come from a package"
+        grouped[axis_order][package].append(item)
+
+    # Flatten back to a list of items
+    new_items: List[pytest.Item] = [
+        item
+        for axis_order in sorted(grouped) # apply axis_ordering here
+        for package in grouped[axis_order]
+        for item in grouped[axis_order][package]
+    ]
+    items[:] = new_items
+
+
 # BEGIN make test results visible from fixtures
 # from https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures