Implement non-default markers

README.md

@@ -202,6 +202,7 @@ We defined various markers, that currently belong to the following conceptual fa
   * Tests that should be run on the largest variety of VMs.
   * Tests that should be run at least once with a very big VM.
 * Markers used in the tests themselves to change their behaviour. Those won't be very useful to select tests with `-m`. We're just mentioning them for the sake of completeness.
+* Some of the above markers are designated as *non-default markers* (see `pytest.ini`). These markers automatically cause the marked tests to be skipped, unless the matching `--enable-<marker>=true` argument is passed to pytest. They are useful for omitting disruptive tests (flaky, slow, require reboot) from the default test suite.
 
 Here's an example of selection we can do thanks to the markers:
 
@@ -217,6 +218,14 @@ Another example:
 pytest tests/uefi_sb -m "multi_vms and unix_vm" --hosts=ip_of_poolmaster --vm=http://path/to/unix_vm_1.xva --vm=http://path/to/unix_vm_2.xva --vm=http://path/to/unix_vm_3.xva
 ```
 
+Example of running reboot tests, which are otherwise skipped by the non-default marker:
+
+```
+# Run the tests/xen suite, including tests that require a reboot
+pytest tests/xen --enable-reboot=true --hosts=ip_of_poolmaster --vm=http://path/to/a_small_vm.xva
+```
+
+Note the difference between `--enable-reboot=true` and `-m reboot`. The former allows the tests marked `reboot` to execute, while the latter specifically selects tests marked `reboot` to be executed. Note that `-m reboot` requires `--enable-reboot=true`.
 
 The `-k` option may also be used to select tests, but this time based on their name. To be used with caution as this can be a fragile filter.
 

conftest.py

@@ -20,6 +20,7 @@
     prefix_object_name,
     setup_formatted_and_mounted_disk,
     shortened_nodeid,
+    strtobool,
     teardown_formatted_and_mounted_disk,
     vm_image,
     wait_for,
@@ -36,7 +37,7 @@
 # need to import them in the global conftest.py so that they are recognized as fixtures.
 from pkgfixtures import formatted_and_mounted_ext4_disk, sr_disk_wiped
 
-from typing import Dict, Generator, Iterable
+from typing import Dict, Generator, Iterable, List
 
 # Do we cache VMs?
 try:
@@ -47,7 +48,9 @@
 
 # pytest hooks
 
-def pytest_addoption(parser):
+NONDEFAULT_MARKERS = ["reboot", "flaky", "slow"]
+
+def pytest_addoption(parser: pytest.Parser):
     parser.addoption(
         "--nest",
         action="store",
@@ -99,12 +102,19 @@ def pytest_addoption(parser):
         help="Format of VDI to execute tests on."
         "Example: vhd,qcow2"
     )
-
-def pytest_configure(config):
+    # Markers
+    for marker in NONDEFAULT_MARKERS:
+        parser.addoption(
+            f"--enable-{marker}",
+            type=strtobool,
+            help=f"Enable tests with the {marker} marker"
+        )
+
+def pytest_configure(config: pytest.Config):
     global_config.ignore_ssh_banner = config.getoption('--ignore-ssh-banner')
     global_config.ssh_output_max_lines = int(config.getoption('--ssh-output-max-lines'))
 
-def pytest_generate_tests(metafunc):
+def pytest_generate_tests(metafunc: pytest.Metafunc):
     if "vm_ref" in metafunc.fixturenames:
         vms = metafunc.config.getoption("vm")
         if not vms:
@@ -117,7 +127,7 @@ def pytest_generate_tests(metafunc):
             image_format = ["vhd"] # Not giving image-format will default to doing tests on vhd
         metafunc.parametrize("image_format", image_format, scope="session")
 
-def pytest_collection_modifyitems(items, config):
+def pytest_collection_modifyitems(items: List[pytest.Item], config: pytest.Config):
     # Automatically mark tests based on fixtures they require.
     # Check pytest.ini or pytest --markers for marker descriptions.
 
@@ -144,6 +154,15 @@ def pytest_collection_modifyitems(items, config):
             # multi_vms implies small_vm
             item.add_marker('small_vm')
 
+    # Disable marked tests if --enable-marker=true is not provided
+    for marker in NONDEFAULT_MARKERS:
+        if not config.getoption(f"--enable-{marker}"):
+            skip = pytest.mark.skip(reason=f"test disabled, pass `--enable-{marker}=true` to enable")
+            for item in items:
+                if marker in item.keywords:
+                    item.add_marker(skip)
+
+
 # BEGIN make test results visible from fixtures
 # from https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures
 

jobs.py

@@ -32,7 +32,7 @@
             "tests/xapi_plugins",
             "tests/install/test_fixtures.py",
         ],
-        "markers": "(small_vm or no_vm) and not flaky and not reboot and not complex_prerequisites",
+        "markers": "(small_vm or no_vm) and not complex_prerequisites",
     },
     "main-multi-unix": {
         "description": "a group of tests that need to run on the largest variety of VMs - unix split",
@@ -47,7 +47,7 @@
             "--vm[]": "multi/all_unix",
         },
         "paths": ["tests/misc", "tests/migration"],
-        "markers": "multi_vms and not flaky and not reboot",
+        "markers": "multi_vms"
     },
     "main-multi-windows": {
         "description": "a group of tests that need to run on the largest variety of VMs - windows split",
@@ -62,7 +62,7 @@
             "--vm[]": "multi/all_windows",
         },
         "paths": ["tests/misc", "tests/migration"],
-        "markers": "multi_vms and not flaky and not reboot",
+        "markers": "multi_vms"
     },
     "packages": {
         "description": "tests that packages can be installed correctly",
@@ -87,7 +87,7 @@
             "--vm": "single/small_vm",
         },
         "paths": ["tests/storage"],
-        "markers": "(small_vm or no_vm) and not reboot and not quicktest and not unused_4k_disks",
+        "markers": "(small_vm or no_vm) and not quicktest and not unused_4k_disks",
         "name_filter": "not migration and not linstor",
     },
     "storage-migrations": {
@@ -117,10 +117,11 @@
         ],
         "nb_pools": 1,
         "params": {
+            "--enable-reboot": True,
             "--vm": "single/small_vm",
         },
         "paths": ["tests/storage"],
-        "markers": "reboot and not flaky and not unused_4k_disks",
+        "markers": "reboot and not unused_4k_disks",
         "name_filter": "not linstor",
     },
     "storage-quicktest": {
@@ -149,7 +150,7 @@
             "--vm": "single/small_vm",
         },
         "paths": ["tests/storage/linstor"],
-        "markers": "(small_vm or no_vm) and not reboot and not quicktest",
+        "markers": "(small_vm or no_vm) and not quicktest",
         "name_filter": "not migration",
     },
     "linstor-migrations": {
@@ -177,6 +178,7 @@
         ],
         "nb_pools": 1,
         "params": {
+            "--enable-reboot": True,
             "--vm": "single/small_vm",
         },
         "paths": ["tests/storage/linstor"],
@@ -206,7 +208,7 @@
             "--vm": "single/small_vm",
         },
         "paths": ["tests/storage"],
-        "markers": "(small_vm or no_vm) and unused_4k_disks and not reboot and not quicktest",
+        "markers": "(small_vm or no_vm) and unused_4k_disks and not quicktest",
         "name_filter": "not migration",
     },
     "largeblock-migrations": {
@@ -234,6 +236,7 @@
         ],
         "nb_pools": 1,
         "params": {
+            "--enable-reboot": True,
             "--vm": "single/small_vm",
         },
         "paths": ["tests/storage"],
@@ -376,6 +379,7 @@
         ],
         "nb_pools": 1,
         "params": {
+            "--enable-reboot": True,
             "--vm": "single/small_vm",
         },
         "paths": ["tests/xen"],
@@ -403,6 +407,8 @@
         ],
         "nb_pools": 1,
         "params": {
+            "--enable-reboot": True,
+            "--enable-flaky": True,
             "--vm": "single/small_vm",
         },
         "paths": ["tests"],
@@ -425,7 +431,9 @@
             "The host will be rebooted by the tests."
         ],
         "nb_pools": 1,
-        "params": {},
+        "params": {
+            "--enable-reboot": True,
+        },
         "paths": ["tests/pci_passthrough"],
     },
     "fs-diff": {
@@ -444,7 +452,9 @@
             "1 XCP-ng pool and an additionnal host >= 8.2"
         ],
         "nb_pools": 2,
-        "params": {},
+        "params": {
+            "--enable-reboot": True,
+        },
         "paths": ["tests/misc/test_pool.py"],
     },
     "limit-tests": {
@@ -458,7 +468,22 @@
             "--vm[]": "multi/limits",
         },
         "paths": ["tests/limits"],
-    }
+    },
+    "slow": {
+        "description": "tests that may take a long time",
+        "requirements": [
+            "Will vary depending on the tests included.",
+            "Use the collect command to get the list of tests "
+            + "and check the requirements written at the top of the test files.",
+        ],
+        "nb_pools": 1,
+        "params": {
+            "--enable-slow": True,
+            "--vm[]": "multi/slow",
+        },
+        "paths": ["tests"],
+        "markers": "slow",
+    },
 }
 
 # List used by the 'check' action: tests listed here will not raise a check error

pytest.ini

@@ -32,9 +32,12 @@ markers =
     multi_vms: tests that it would be good to run on a variety of VMs (includes `small_vm` but excludes `big_vm`).
     debian_uefi_vm: tests that require a Debian UEFI VM
 
-    # * Other markers
+    # * Non-default markers
     reboot: tests that reboot one or more hosts.
     flaky: flaky tests. Usually pass, but sometimes fail unexpectedly.
+    slow: tests that may take a long time, designed to catch rare issues.
+
+    # * Other markers
     complex_prerequisites: tests whose prerequisites are complex and may require special attention.
     quicktest: runs `quicktest`.
 log_level = debug

tests/misc/test_vm_basic_operations.py

@@ -19,6 +19,19 @@ def test_suspend(self, running_vm: VM):
         vm.resume()
         vm.wait_for_vm_running_and_ssh_up()
 
+    @pytest.mark.slow
+    def test_migrate_repeat(self, running_vm: VM):
+        """
+        Perform a "fast" stress test of the suspend mechanism by repeatedly migrating onto the same host.
+
+        We don't use the usual suspend call, but use local migrations to avoid hitting the disk and/or network.
+        """
+        vm = running_vm
+        residence = vm.get_residence_host()
+        for _attempt in range(100):
+            vm.migrate(residence)
+            vm.wait_for_vm_running_and_ssh_up()
+
     def test_snapshot(self, running_vm: VM):
         vm = running_vm
         vm.test_snapshot_on_running_vm()

tests/storage/iso/test_nfs_iso_sr.py

@@ -35,6 +35,12 @@ def test_create_and_destroy_sr(self, host, nfs_iso_device_config):
         sr = host.sr_create('iso', "ISO-NFS-SR-test", nfs_iso_device_config, shared=True, verify=True)
         sr.forget()
 
+    @pytest.mark.slow
+    def test_create_and_destroy_sr_repeat(self, host, nfs_iso_device_config):
+        """Look for potential issues in the NFS subsystem by stress testing the connection and disconnection code."""
+        for _attempt in range(100):
+            self.test_create_and_destroy_sr(host, nfs_iso_device_config)
+
 
 @pytest.mark.small_vm
 @pytest.mark.usefixtures("nfs_iso_sr")

vm_data.py-dist

@@ -65,6 +65,11 @@ VMS["multi"]["limits"] = [vm for vm in [
     VMS["single"]["small_vm_windows"],
 ] if vm]
 
+VMS["multi"]["slow"] = [vm for vm in [
+    VMS["single"]["small_vm"],
+    VMS["single"]["small_vm_windows"],
+] if vm]
+
 # Example of use for a common XVA_LOCATION
 #
 # XVA_LOCATION="http://somewhere/"