storage: add jobs for large volume tests

Mark LVM and lvmoiscsi tests that allocate large VDIs with disk_space_allocation,
and all test_full_vdi_write tests with disk_throughput_intensive, to allow running
large-VDI jobs only on hardware that can handle the respective disk requirements.

Add storage-main-16tb-{thin,thick,full-write} and storage-migrations-16tb-{thin,thick}
jobs that use these markers to split 16TiB volume testing by disk speed requirement.

Signed-off-by: Gaëtan Lehmann <gaetan.lehmann@vates.tech>

Author: glehmann

jobs.py

@@ -101,6 +101,62 @@ class JobData(TypedDict):
         "markers": "(small_vm or no_vm) and not reboot and not quicktest and not unused_4k_disks",
         "name_filter": "not migration and not linstor",
     },
+    "storage-main-large-thin": {
+        "description":
+            "same as storage-main with 3TiB VDIs, no large gzip XVAs creation and no test requiring 3TiB allocation",
+        "requirements": [
+            "A pool with at least 3 hosts.",
+            "An additional free disk on every host.",
+            "Configuration in data.py for each remote SR that will be tested.",
+            "A small VM that can be imported on the SRs.",
+        ],
+        "nb_pools": 1,
+        "params": {
+            "--vm": "single/small_vm",
+            "--volume-size": "3TiB",
+        },
+        "paths": ["tests/storage"],
+        "markers": "(small_vm or no_vm) and not reboot and not quicktest and not unused_4k_disks"
+                   " and not thick_provisioned and not disk_throughput_intensive",
+        "name_filter": "not migration and not linstor and not gzip",
+    },
+    "storage-main-large-thick": {
+        "description":
+            "same as storage-main with 3TiB VDIs, no large gzip XVAs creation and tests requiring 3TiB allocation",
+        "requirements": [
+            "A pool with at least 3 hosts.",
+            "An additional free disk on every host.",
+            "Configuration in data.py for each remote SR that will be tested.",
+            "A small VM that can be imported on the SRs.",
+        ],
+        "nb_pools": 1,
+        "params": {
+            "--vm": "single/small_vm",
+            "--volume-size": "3TiB",
+        },
+        "paths": ["tests/storage"],
+        "markers": "(small_vm or no_vm) and not reboot and not quicktest and not unused_4k_disks"
+                   " and not disk_throughput_intensive and thick_provisioned",
+        "name_filter": "not migration and not linstor and not gzip",
+    },
+    "storage-main-large-full-write": {
+        "description": "storage tests actually writing the full 3TiB volumes",
+        "requirements": [
+            "A pool with at least 3 hosts.",
+            "An additional free disk on every host.",
+            "Configuration in data.py for each remote SR that will be tested.",
+            "A small VM that can be imported on the SRs.",
+        ],
+        "nb_pools": 1,
+        "params": {
+            "--vm": "single/small_vm",
+            "--volume-size": "3TiB",
+        },
+        "paths": ["tests/storage"],
+        "markers": "(small_vm or no_vm) and not reboot and not quicktest and not unused_4k_disks"
+                   " and disk_throughput_intensive",
+        "name_filter": "not migration and not linstor and not gzip",
+    },
     "storage-migrations": {
         "description": "tests migrations with all storage drivers",
         "requirements": [
@@ -118,6 +174,42 @@ class JobData(TypedDict):
         "markers": "not unused_4k_disks",
         "name_filter": "migration and not linstor",
     },
+    "storage-migrations-large-thin": {
+        "description": "same as storage-migrations with 3TiB VDIs, and no test requiring 3TiB allocation",
+        "requirements": [
+            "A pool with at least 3 hosts.",
+            "An additional free disk on every host.",
+            "A second pool with at least 1 host and a SR to receive VMs.",
+            "Configuration in data.py for each remote SR that will be tested.",
+            "A small VM that can be imported on the SRs.",
+        ],
+        "nb_pools": 2,
+        "params": {
+            "--vm": "single/small_vm",
+            "--volume-size": "3TiB",
+        },
+        "paths": ["tests/storage"],
+        "markers": "not unused_4k_disks and not thick_provisioned",
+        "name_filter": "migration and not linstor",
+    },
+    "storage-migrations-large-thick": {
+        "description": "same as storage-migrations with 3TiB VDIs, and tests requiring 3TiB allocation",
+        "requirements": [
+            "A pool with at least 3 hosts.",
+            "An additional free disk on every host.",
+            "A second pool with at least 1 host and a SR to receive VMs.",
+            "Configuration in data.py for each remote SR that will be tested.",
+            "A small VM that can be imported on the SRs.",
+        ],
+        "nb_pools": 2,
+        "params": {
+            "--vm": "single/small_vm",
+            "--volume-size": "3TiB",
+        },
+        "paths": ["tests/storage"],
+        "markers": "not unused_4k_disks and thick_provisioned",
+        "name_filter": "migration and not linstor",
+    },
     "storage-reboots": {
         "description": "storage driver tests that involve rebooting hosts (except flaky tests)",
         "requirements": [

pytest.ini

@@ -32,6 +32,10 @@ markers =
     multi_vms: tests that it would be good to run on a variety of VMs (includes `small_vm` but excludes `big_vm`).
     debian_uefi_vm: tests that require a Debian UEFI VM
 
+    # * Disk-related markers
+    thick_provisioned: tests that use thick provisioned SRs.
+    disk_throughput_intensive: tests that fill a large VDI entirely (require a fast disk).
+
     # * Other markers
     reboot: tests that reboot one or more hosts.
     flaky: flaky tests. Usually pass, but sometimes fail unexpectedly.

tests/storage/ext/test_ext_sr.py

@@ -104,6 +104,7 @@ def test_vdi_export_import(self, storage_test_vm: VM, ext_sr: SR, image_format:
         vdi_export_import(storage_test_vm, ext_sr, image_format, temp_large_dir, defer)
 
     @pytest.mark.small_vm
+    @pytest.mark.disk_throughput_intensive
     def test_full_vdi_write(self, storage_test_vm: VM, vdi_on_ext_sr: VDI, defer: Defer):
         full_vdi_write(storage_test_vm, vdi_on_ext_sr, defer)
 

tests/storage/lvm/test_lvm_sr.py

@@ -54,12 +54,13 @@ def test_create_and_destroy_sr(self, host: Host,
         sr.destroy(verify=True)
 
 @pytest.mark.usefixtures("lvm_sr")
+@pytest.mark.thick_provisioned
 class TestLVMSR:
     @pytest.mark.quicktest
     def test_quicktest(self, lvm_sr: SR) -> None:
         lvm_sr.run_quicktest()
 
-    def test_vdi_is_not_open(self, vdi_on_lvm_sr: VDI) -> None:
+    def test_vdi_is_not_open(self, vdi_on_lvm_sr) -> None:
         assert not vdi_is_open(vdi_on_lvm_sr)
 
     def test_vdi_image_format(self, vdi_on_lvm_sr: VDI, image_format: ImageFormat) -> None:
@@ -150,6 +151,7 @@ def test_coalesce(self, storage_test_vm: VM, vdi_on_lvm_sr: VDI, vdi_op: Coalesc
         coalesce_integrity(storage_test_vm, vdi_on_lvm_sr, vdi_op, defer)
 
     @pytest.mark.small_vm
+    @pytest.mark.disk_throughput_intensive
     def test_full_vdi_write(self, storage_test_vm: VM, vdi_on_lvm_sr: VDI, defer: Defer):
         full_vdi_write(storage_test_vm, vdi_on_lvm_sr, defer)
 

tests/storage/lvm/test_lvm_sr_crosspool_migration.py

@@ -15,6 +15,7 @@
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
 @pytest.mark.usefixtures("hostB1", "local_sr_on_hostB1")
+@pytest.mark.thick_provisioned
 class Test:
     def test_cold_crosspool_migration(self, host: Host, hostB1: Host, vm_on_lvm_sr: VM, local_sr_on_hostB1: SR) -> None:
         cold_migration_then_come_back(vm_on_lvm_sr, host, hostB1, local_sr_on_hostB1)

tests/storage/lvm/test_lvm_sr_intrapool_migration.py

@@ -15,6 +15,7 @@
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
 @pytest.mark.usefixtures("hostA2", "local_sr_on_hostA2")
+@pytest.mark.thick_provisioned
 class Test:
     def test_cold_intrapool_migration(self, host: Host, hostA2: Host, vm_on_lvm_sr: VM, local_sr_on_hostA2: SR) -> None:
         cold_migration_then_come_back(vm_on_lvm_sr, host, hostA2, local_sr_on_hostA2)

tests/storage/lvmoiscsi/test_lvmoiscsi_sr.py

@@ -43,12 +43,13 @@ def test_create_and_destroy_sr(self, host: Host, lvmoiscsi_device_config: dict[s
 
 @pytest.mark.usefixtures('image_format')
 @pytest.mark.usefixtures("lvmoiscsi_sr")
+@pytest.mark.thick_provisioned
 class TestLVMOISCSISR:
     @pytest.mark.quicktest
     def test_quicktest(self, lvmoiscsi_sr: SR) -> None:
         lvmoiscsi_sr.run_quicktest()
 
-    def test_vdi_is_not_open(self, vdi_on_lvmoiscsi_sr: VDI) -> None:
+    def test_vdi_is_not_open(self, vdi_on_lvmoiscsi_sr) -> None:
         assert not vdi_is_open(vdi_on_lvmoiscsi_sr)
 
     @pytest.mark.small_vm # run with a small VM to test the features
@@ -77,6 +78,7 @@ def test_coalesce(self, storage_test_vm: 'VM', vdi_on_lvmoiscsi_sr: 'VDI', vdi_o
         coalesce_integrity(storage_test_vm, vdi_on_lvmoiscsi_sr, vdi_op, defer)
 
     @pytest.mark.small_vm
+    @pytest.mark.disk_throughput_intensive
     def test_full_vdi_write(self, storage_test_vm: VM, vdi_on_lvmoiscsi_sr: VDI, defer: Defer):
         full_vdi_write(storage_test_vm, vdi_on_lvmoiscsi_sr, defer)
 

tests/storage/lvmoiscsi/test_lvmoiscsi_sr_crosspool_migration.py

@@ -15,6 +15,7 @@
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
 @pytest.mark.usefixtures("hostB1", "local_sr_on_hostB1")
+@pytest.mark.thick_provisioned
 class Test:
     def test_cold_crosspool_migration(
         self, host: Host, hostB1: Host, vm_on_lvmoiscsi_sr: VM, local_sr_on_hostB1: SR

tests/storage/lvmoiscsi/test_lvmoiscsi_sr_intrapool_migration.py

@@ -15,6 +15,7 @@
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
 @pytest.mark.usefixtures("hostA2", "local_sr_on_hostA2")
+@pytest.mark.thick_provisioned
 class Test:
     def test_live_intrapool_shared_migration(self, host: Host, hostA2: Host, vm_on_lvmoiscsi_sr: VM) -> None:
         sr = vm_on_lvmoiscsi_sr.get_sr()

tests/storage/nfs/test_nfs_sr.py

@@ -122,6 +122,7 @@ def test_coalesce(self, storage_test_vm: VM, dispatch_nfs: VDI, vdi_op: Coalesce
         coalesce_integrity(storage_test_vm, dispatch_nfs, vdi_op, defer)
 
     @pytest.mark.small_vm
+    @pytest.mark.disk_throughput_intensive
     def test_full_vdi_write(self, storage_test_vm: VM, vdi_on_nfs_sr: VDI, defer: Defer):
         full_vdi_write(storage_test_vm, vdi_on_nfs_sr, defer)