Add explicit enable switch for reboot/flaky markers

This avoids us having to specify 'not reboot/flaky' everywhere in
jobs.py by adding an implicit skip marker if the corresponding arguments
are not specified in the pytest call.

Signed-off-by: Tu Dinh <ngoc-tu.dinh@vates.tech>

Author: dinhngtu

conftest.py

@@ -20,6 +20,7 @@
     prefix_object_name,
     setup_formatted_and_mounted_disk,
     shortened_nodeid,
+    strtobool,
     teardown_formatted_and_mounted_disk,
     vm_image,
     wait_for,
@@ -47,6 +48,8 @@
 
 # pytest hooks
 
+NONDEFAULT_MARKERS = ["reboot", "flaky"]
+
 def pytest_addoption(parser: pytest.Parser):
     parser.addoption(
         "--nest",
@@ -99,6 +102,13 @@ def pytest_addoption(parser: pytest.Parser):
         help="Format of VDI to execute tests on."
         "Example: vhd,qcow2"
     )
+    # Markers
+    for marker in NONDEFAULT_MARKERS:
+        parser.addoption(
+            f"--enable-{marker}",
+            type=strtobool,
+            help=f"Enable tests with the {marker} marker"
+        )
 
 def pytest_configure(config: pytest.Config):
     global_config.ignore_ssh_banner = config.getoption('--ignore-ssh-banner')
@@ -144,6 +154,15 @@ def pytest_collection_modifyitems(items: List[pytest.Item], config: pytest.Confi
             # multi_vms implies small_vm
             item.add_marker('small_vm')
 
+    # Disable marked tests if --enable-marker=true is not provided
+    for marker in NONDEFAULT_MARKERS:
+        if not config.getoption(f"--enable-{marker}"):
+            skip = pytest.mark.skip(reason="test disabled, pass `--enable-{marker}=true` to enable")
+            for item in items:
+                if marker in item.keywords:
+                    item.add_marker(skip)
+
+
 # BEGIN make test results visible from fixtures
 # from https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures
 

jobs.py

@@ -32,7 +32,7 @@
             "tests/xapi_plugins",
             "tests/install/test_fixtures.py",
         ],
-        "markers": "(small_vm or no_vm) and not flaky and not reboot and not complex_prerequisites",
+        "markers": "(small_vm or no_vm) and not complex_prerequisites",
     },
     "main-multi-unix": {
         "description": "a group of tests that need to run on the largest variety of VMs - unix split",
@@ -47,7 +47,7 @@
             "--vm[]": "multi/all_unix",
         },
         "paths": ["tests/misc", "tests/migration"],
-        "markers": "multi_vms and not flaky and not reboot",
+        "markers": "multi_vms"
     },
     "main-multi-windows": {
         "description": "a group of tests that need to run on the largest variety of VMs - windows split",
@@ -62,7 +62,7 @@
             "--vm[]": "multi/all_windows",
         },
         "paths": ["tests/misc", "tests/migration"],
-        "markers": "multi_vms and not flaky and not reboot",
+        "markers": "multi_vms"
     },
     "packages": {
         "description": "tests that packages can be installed correctly",
@@ -87,7 +87,7 @@
             "--vm": "single/small_vm",
         },
         "paths": ["tests/storage"],
-        "markers": "(small_vm or no_vm) and not reboot and not quicktest and not unused_4k_disks",
+        "markers": "(small_vm or no_vm) and not quicktest and not unused_4k_disks",
         "name_filter": "not migration and not linstor",
     },
     "storage-migrations": {
@@ -117,10 +117,11 @@
         ],
         "nb_pools": 1,
         "params": {
+            "--enable-reboot": True,
             "--vm": "single/small_vm",
         },
         "paths": ["tests/storage"],
-        "markers": "reboot and not flaky and not unused_4k_disks",
+        "markers": "reboot and not unused_4k_disks",
         "name_filter": "not linstor",
     },
     "storage-quicktest": {
@@ -149,7 +150,7 @@
             "--vm": "single/small_vm",
         },
         "paths": ["tests/storage/linstor"],
-        "markers": "(small_vm or no_vm) and not reboot and not quicktest",
+        "markers": "(small_vm or no_vm) and not quicktest",
         "name_filter": "not migration",
     },
     "linstor-migrations": {
@@ -177,6 +178,7 @@
         ],
         "nb_pools": 1,
         "params": {
+            "--enable-reboot": True,
             "--vm": "single/small_vm",
         },
         "paths": ["tests/storage/linstor"],
@@ -206,7 +208,7 @@
             "--vm": "single/small_vm",
         },
         "paths": ["tests/storage"],
-        "markers": "(small_vm or no_vm) and unused_4k_disks and not reboot and not quicktest",
+        "markers": "(small_vm or no_vm) and unused_4k_disks and not quicktest",
         "name_filter": "not migration",
     },
     "largeblock-migrations": {
@@ -234,6 +236,7 @@
         ],
         "nb_pools": 1,
         "params": {
+            "--enable-reboot": True,
             "--vm": "single/small_vm",
         },
         "paths": ["tests/storage"],
@@ -376,6 +379,7 @@
         ],
         "nb_pools": 1,
         "params": {
+            "--enable-reboot": True,
             "--vm": "single/small_vm",
         },
         "paths": ["tests/xen"],
@@ -403,6 +407,8 @@
         ],
         "nb_pools": 1,
         "params": {
+            "--enable-reboot": True,
+            "--enable-flaky": True,
             "--vm": "single/small_vm",
         },
         "paths": ["tests"],
@@ -425,7 +431,9 @@
             "The host will be rebooted by the tests."
         ],
         "nb_pools": 1,
-        "params": {},
+        "params": {
+            "--enable-reboot": True,
+        },
         "paths": ["tests/pci_passthrough"],
     },
     "fs-diff": {
@@ -444,7 +452,9 @@
             "1 XCP-ng pool and an additionnal host >= 8.2"
         ],
         "nb_pools": 2,
-        "params": {},
+        "params": {
+            "--enable-reboot": True,
+        },
         "paths": ["tests/misc/test_pool.py"],
     },
     "limit-tests": {