jobs: avoid running tests running hosts_with_xo in 'main' jobs

With traffic rules tests, we are adding more tests depending on XO (traffic 
rules are managed by XO).

In order to avoid this strong dependency in 'main' jobs, automatically mark 
tests using 'hosts_with_xo' fixture with 'hosts_with_xo' mark. It permits to 
avoid running them in 'main' jobs, and run them in 'advanced-network' jobs 
instead of (for networks tests).

Additionally, restrict the use of xo_cli() function to tests that are using 
'hosts_with_xo' fixture.

It is done by modifying a global in lib.xo module, and checking it at xo_cli() 
beginning.

Based on idea from @stormi

Signed-off-by: Sebastien Rodot <sebastien.rodot@vates.tech>

Author: semarie

conftest.py

@@ -37,7 +37,7 @@
 from lib.vbd import VBD
 from lib.vdi import VDI
 from lib.vm import VM, vm_cache_key_from_def
-from lib.xo import xo_cli
+from lib.xo import _allow_xo_cli, xo_cli
 
 # Import package-scoped fixtures. Although we need to define them in a separate file so that we can
 # then import them in individual packages to fix the buggy package scope handling by pytest, we also
@@ -202,6 +202,7 @@ def pytest_collection_modifyitems(items: list[pytest.Item], config: pytest.Confi
         'hostB1',
         'unused_512B_disks',
         'unused_4k_disks',
+        'hosts_with_xo',
     ]
 
     # -------------
@@ -367,12 +368,15 @@ def registered_xo_cli() -> None:
     # The fixture is not responsible for establishing the connection.
     # We just check that xo-cli is currently registered
     try:
+        old_allow_xo_cli = _allow_xo_cli(True)
         xo_cli('server.getAll')
+        _allow_xo_cli(old_allow_xo_cli)
     except Exception as e:
-        raise Exception(f"Check for registered xo_cli failed: {e}")
+        pytest.fail(f"Check for registered xo_cli failed: {e}")
 
 @pytest.fixture(scope='session')
 def hosts_with_xo(hosts: list[Host], registered_xo_cli: None) -> Generator[list[Host], None, None]:
+    old_allow_xo_cli = _allow_xo_cli(True)
     for h in hosts:
         logging.info(">>> Connect host %s" % h)
         if not h.skip_xo_config:
@@ -386,6 +390,7 @@ def hosts_with_xo(hosts: list[Host], registered_xo_cli: None) -> Generator[list[
         if not h.skip_xo_config:
             logging.info("<<< Disconnect host %s" % h)
             h.xo_server_remove()
+    _allow_xo_cli(old_allow_xo_cli)
 
 @pytest.fixture(scope='session')
 def hostA1(hosts: list[Host]) -> Generator[Host, None, None]:

jobs.py

@@ -44,7 +44,8 @@ class JobData(TypedDict):
             "tests/xapi_plugins",
             "tests/install/test_fixtures.py",
         ],
-        "markers": "(small_vm or no_vm) and not flaky and not reboot and not complex_prerequisites",
+        "markers": "(small_vm or no_vm) and not flaky and not reboot "
+        "and not hosts_with_xo and not complex_prerequisites",
     },
     "main-multi-unix": {
         "description": "a group of tests that need to run on the largest variety of VMs - unix split",
@@ -82,13 +83,14 @@ class JobData(TypedDict):
             "A pool with at least 1 host (if more, with same network configuration).",
             "At least 2 free NICs on every host.",
             "A small VM that can be imported on the SRs.",
+            "xo-cli locally installed, in $PATH, and registered to an XO instance.",
         ],
         "nb_pools": 1,
         "params": {
             "--vm": "single/small_vm",
         },
         "paths": ["tests/network"],
-        "markers": "complex_prerequisites",
+        "markers": "complex_prerequisites or hosts_with_xo",
     },
     "packages": {
         "description": "tests that packages can be installed correctly",

lib/xo.py

@@ -1,3 +1,5 @@
+import pytest
+
 import json
 
 from data import TOOLS
@@ -6,6 +8,20 @@
 
 from typing import Literal, overload
 
+__allow_xo_cli = False
+def _allow_xo_cli(value: bool) -> bool:
+    """
+    Permit to configure the usage of xo_cli function (returns the previous value).
+    This function shoudln't be called directly.
+    If you need xo_cli(), use the hosts_with_xo fixture.
+    """
+    global __allow_xo_cli
+
+    old = __allow_xo_cli
+    __allow_xo_cli = value
+
+    return old
+
 @overload
 def xo_cli(action: str, args: dict[str, str] = {}, *, check: bool = True, use_json: Literal[False] = False) -> str:
     ...
@@ -14,6 +30,9 @@ def xo_cli(action: str, args: dict[str, str] = {}, *, check: bool = True, use_js
     ...
 
 def xo_cli(action: str, args: dict[str, str] = {}, *, check: bool = True, use_json: bool = False) -> JSONType | str:
+    if not __allow_xo_cli:
+        pytest.fail("xo_cli function requires hosts_with_xo fixture usage.")
+
     cmd = [TOOLS.get('xo-cli', 'xo-cli'), action]
     if action != 'list-objects' and use_json:
         cmd += ['--json']

pytest.ini

@@ -38,6 +38,7 @@ markers =
 
     # * Other markers
     reboot: tests that reboot one or more hosts.
+    hosts_with_xo: tests that require XO to be available via xo-cli tool.
     flaky: flaky tests. Usually pass, but sometimes fail unexpectedly.
     complex_prerequisites: tests whose prerequisites are complex and may require special attention.
     quicktest: runs `quicktest`.