storage: add jobs for large volume tests

Mark LVM and lvmoiscsi tests that allocate large VDIs with disk_space_allocation,
and all test_full_vdi_write tests with disk_throughput_intensive, to allow running
large-VDI jobs only on hardware that can handle the respective disk requirements.

Add storage-main-16tb-{thin,thick,full-write} and storage-migrations-16tb-{thin,thick}
jobs that use these markers to split 16TiB volume testing by disk speed requirement.

Signed-off-by: Gaëtan Lehmann <gaetan.lehmann@vates.tech>

Author: glehmann

jobs.py

@@ -90,6 +90,62 @@
         "markers": "(small_vm or no_vm) and not reboot and not quicktest and not unused_4k_disks",
         "name_filter": "not migration and not linstor",
     },
+    "storage-main-16tb-thin": {
+        "description":
+            "same as storage-main with 16TiB VDIs, no gzip XVAs and no test requiring 16TiB allocation",
+        "requirements": [
+            "A pool with at least 3 hosts.",
+            "An additional free disk on every host.",
+            "Configuration in data.py for each remote SR that will be tested.",
+            "A small VM that can be imported on the SRs.",
+        ],
+        "nb_pools": 1,
+        "params": {
+            "--vm": "single/small_vm",
+            "--volume-size": "16TiB",
+        },
+        "paths": ["tests/storage"],
+        "markers": "(small_vm or no_vm) and not reboot and not quicktest and not unused_4k_disks"
+                   " and not disk_space_allocation and not disk_throughput_intensive",
+        "name_filter": "not migration and not linstor and not gzip",
+    },
+    "storage-main-16tb-thick": {
+        "description":
+            "same as storage-main with 16TiB VDIs, no gzip XVAs and tests requiring 16TiB allocation",
+        "requirements": [
+            "A pool with at least 3 hosts.",
+            "An additional free disk on every host.",
+            "Configuration in data.py for each remote SR that will be tested.",
+            "A small VM that can be imported on the SRs.",
+        ],
+        "nb_pools": 1,
+        "params": {
+            "--vm": "single/small_vm",
+            "--volume-size": "16TiB",
+        },
+        "paths": ["tests/storage"],
+        "markers": "(small_vm or no_vm) and not reboot and not quicktest and not unused_4k_disks"
+                   " and not disk_throughput_intensive and disk_space_allocation",
+        "name_filter": "not migration and not linstor and not gzip",
+    },
+    "storage-main-16tb-full-write": {
+        "description": "storage tests actually writing the full 16TiB volumes",
+        "requirements": [
+            "A pool with at least 3 hosts.",
+            "An additional free disk on every host.",
+            "Configuration in data.py for each remote SR that will be tested.",
+            "A small VM that can be imported on the SRs.",
+        ],
+        "nb_pools": 1,
+        "params": {
+            "--vm": "single/small_vm",
+            "--volume-size": "16TiB",
+        },
+        "paths": ["tests/storage"],
+        "markers": "(small_vm or no_vm) and not reboot and not quicktest and not unused_4k_disks"
+                   " and disk_throughput_intensive",
+        "name_filter": "not migration and not linstor and not gzip",
+    },
     "storage-migrations": {
         "description": "tests migrations with all storage drivers",
         "requirements": [
@@ -107,6 +163,42 @@
         "markers": "not unused_4k_disks",
         "name_filter": "migration and not linstor",
     },
+    "storage-migrations-16tb-thin": {
+        "description": "same as storage-migrations with 16TiB VDIs, and no test requiring 16TiB allocation",
+        "requirements": [
+            "A pool with at least 3 hosts.",
+            "An additional free disk on every host.",
+            "A second pool with at least 1 host and a SR to receive VMs.",
+            "Configuration in data.py for each remote SR that will be tested.",
+            "A small VM that can be imported on the SRs.",
+        ],
+        "nb_pools": 2,
+        "params": {
+            "--vm": "single/small_vm",
+            "--volume-size": "16TiB",
+        },
+        "paths": ["tests/storage"],
+        "markers": "not unused_4k_disks and not disk_space_allocation",
+        "name_filter": "migration and not linstor",
+    },
+    "storage-migrations-16tb-thick": {
+        "description": "same as storage-migrations with 16TiB VDIs, and tests requiring 16TiB allocation",
+        "requirements": [
+            "A pool with at least 3 hosts.",
+            "An additional free disk on every host.",
+            "A second pool with at least 1 host and a SR to receive VMs.",
+            "Configuration in data.py for each remote SR that will be tested.",
+            "A small VM that can be imported on the SRs.",
+        ],
+        "nb_pools": 2,
+        "params": {
+            "--vm": "single/small_vm",
+            "--volume-size": "16TiB",
+        },
+        "paths": ["tests/storage"],
+        "markers": "not unused_4k_disks and disk_space_allocation",
+        "name_filter": "migration and not linstor",
+    },
     "storage-reboots": {
         "description": "storage driver tests that involve rebooting hosts (except flaky tests)",
         "requirements": [

pytest.ini

@@ -32,6 +32,10 @@ markers =
     multi_vms: tests that it would be good to run on a variety of VMs (includes `small_vm` but excludes `big_vm`).
     debian_uefi_vm: tests that require a Debian UEFI VM
 
+    # * Disk-related markers
+    disk_space_allocation: tests that allocate a large VDI (disk won't be fully written, slow disk is OK).
+    disk_throughput_intensive: tests that fill a large VDI entirely (require a fast disk).
+
     # * Other markers
     reboot: tests that reboot one or more hosts.
     flaky: flaky tests. Usually pass, but sometimes fail unexpectedly.

tests/storage/ext/test_ext_sr.py

@@ -106,6 +106,7 @@ def test_vdi_export_import(
         vdi_export_import(storage_test_vm, ext_sr, image_format, temp_large_dir, defer)
 
     @pytest.mark.small_vm
+    @pytest.mark.disk_throughput_intensive
     def test_full_vdi_write(self, storage_test_vm: VM, vdi_on_ext_sr: VDI, defer: Defer):
         full_vdi_write(storage_test_vm, vdi_on_ext_sr, defer)
 

tests/storage/lvm/test_lvm_sr.py

@@ -59,9 +59,11 @@ class TestLVMSR:
     def test_quicktest(self, lvm_sr):
         lvm_sr.run_quicktest()
 
+    @pytest.mark.disk_space_allocation
     def test_vdi_is_not_open(self, vdi_on_lvm_sr):
         assert not vdi_is_open(vdi_on_lvm_sr)
 
+    @pytest.mark.disk_space_allocation
     def test_vdi_image_format(self, vdi_on_lvm_sr: VDI, image_format: ImageFormat):
         fmt = vdi_on_lvm_sr.get_image_format()
         # feature-detect: if the SM doesn't report image-format, skip this check
@@ -142,12 +144,15 @@ def test_failing_resize_on_inflate_after_setSizePhys(self, host, lvm_sr, vm_on_l
 
     @pytest.mark.small_vm
     @pytest.mark.parametrize("vdi_op", ["snapshot", "clone"])
+    @pytest.mark.disk_space_allocation
     def test_coalesce(
         self, storage_test_vm: VM, vdi_on_lvm_sr: VDI, vdi_op: CoalesceOperation, defer: Defer
     ):
         coalesce_integrity(storage_test_vm, vdi_on_lvm_sr, vdi_op, defer)
 
     @pytest.mark.small_vm
+    @pytest.mark.disk_space_allocation
+    @pytest.mark.disk_throughput_intensive
     def test_full_vdi_write(self, storage_test_vm: VM, vdi_on_lvm_sr: VDI, defer: Defer):
         full_vdi_write(storage_test_vm, vdi_on_lvm_sr, defer)
 
@@ -159,10 +164,12 @@ def test_invalid_vdi_size(self, lvm_sr: SR, image_format: ImageFormat):
 
     @pytest.mark.small_vm
     @pytest.mark.parametrize("compression", ["none", "gzip", "zstd"])
+    @pytest.mark.disk_space_allocation
     def test_xva_export_import(self, vm_on_lvm_sr: VM, compression: XVACompression, temp_large_dir: str, defer: Defer):
         xva_export_import(vm_on_lvm_sr, compression, temp_large_dir, defer)
 
     @pytest.mark.small_vm
+    @pytest.mark.disk_space_allocation
     def test_vdi_export_import(
         self, storage_test_vm: VM, lvm_sr: SR, image_format: ImageFormat, temp_large_dir: str, defer: Defer
     ):

tests/storage/lvm/test_lvm_sr_crosspool_migration.py

@@ -12,6 +12,7 @@
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
 @pytest.mark.usefixtures("hostB1", "local_sr_on_hostB1")
+@pytest.mark.disk_space_allocation
 class Test:
     def test_cold_crosspool_migration(self, host, hostB1, vm_on_lvm_sr, local_sr_on_hostB1):
         cold_migration_then_come_back(vm_on_lvm_sr, host, hostB1, local_sr_on_hostB1)

tests/storage/lvm/test_lvm_sr_intrapool_migration.py

@@ -12,6 +12,7 @@
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
 @pytest.mark.usefixtures("hostA2", "local_sr_on_hostA2")
+@pytest.mark.disk_space_allocation
 class Test:
     def test_cold_intrapool_migration(self, host, hostA2, vm_on_lvm_sr, local_sr_on_hostA2):
         cold_migration_then_come_back(vm_on_lvm_sr, host, hostA2, local_sr_on_hostA2)

tests/storage/lvmoiscsi/test_lvmoiscsi_sr.py

@@ -43,6 +43,7 @@ class TestLVMOISCSISR:
     def test_quicktest(self, lvmoiscsi_sr):
         lvmoiscsi_sr.run_quicktest()
 
+    @pytest.mark.disk_space_allocation
     def test_vdi_is_not_open(self, vdi_on_lvmoiscsi_sr):
         assert not vdi_is_open(vdi_on_lvmoiscsi_sr)
 
@@ -67,6 +68,7 @@ def test_snapshot(self, vm_on_lvmoiscsi_sr):
 
     @pytest.mark.small_vm
     @pytest.mark.parametrize("vdi_op", ["snapshot", "clone"])
+    @pytest.mark.disk_space_allocation
     def test_coalesce(
         self,
         storage_test_vm: 'VM',
@@ -77,6 +79,8 @@ def test_coalesce(
         coalesce_integrity(storage_test_vm, vdi_on_lvmoiscsi_sr, vdi_op, defer)
 
     @pytest.mark.small_vm
+    @pytest.mark.disk_space_allocation
+    @pytest.mark.disk_throughput_intensive
     def test_full_vdi_write(self, storage_test_vm: VM, vdi_on_lvmoiscsi_sr: VDI, defer: Defer):
         full_vdi_write(storage_test_vm, vdi_on_lvmoiscsi_sr, defer)
 
@@ -88,12 +92,14 @@ def test_invalid_vdi_size(self, lvmoiscsi_sr: SR, image_format: ImageFormat):
 
     @pytest.mark.small_vm
     @pytest.mark.parametrize("compression", ["none", "gzip", "zstd"])
+    @pytest.mark.disk_space_allocation
     def test_xva_export_import(
         self, vm_on_lvmoiscsi_sr: VM, compression: XVACompression, temp_large_dir: str, defer: Defer
     ):
         xva_export_import(vm_on_lvmoiscsi_sr, compression, temp_large_dir, defer)
 
     @pytest.mark.small_vm
+    @pytest.mark.disk_space_allocation
     def test_vdi_export_import(
         self, storage_test_vm: VM, lvmoiscsi_sr: SR, image_format: ImageFormat, temp_large_dir: str, defer: Defer
     ):

tests/storage/lvmoiscsi/test_lvmoiscsi_sr_crosspool_migration.py

@@ -12,6 +12,7 @@
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
 @pytest.mark.usefixtures("hostB1", "local_sr_on_hostB1")
+@pytest.mark.disk_space_allocation
 class Test:
     def test_cold_crosspool_migration(self, host, hostB1, vm_on_lvmoiscsi_sr, local_sr_on_hostB1):
         cold_migration_then_come_back(vm_on_lvmoiscsi_sr, host, hostB1, local_sr_on_hostB1)

tests/storage/lvmoiscsi/test_lvmoiscsi_sr_intrapool_migration.py

@@ -12,6 +12,7 @@
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
 @pytest.mark.usefixtures("hostA2", "local_sr_on_hostA2")
+@pytest.mark.disk_space_allocation
 class Test:
     def test_live_intrapool_shared_migration(self, host, hostA2, vm_on_lvmoiscsi_sr):
         sr = vm_on_lvmoiscsi_sr.get_sr()

tests/storage/nfs/test_nfs_sr.py

@@ -123,6 +123,7 @@ def test_coalesce(
         coalesce_integrity(storage_test_vm, dispatch_nfs, vdi_op, defer)
 
     @pytest.mark.small_vm
+    @pytest.mark.disk_throughput_intensive
     def test_full_vdi_write(self, storage_test_vm: VM, vdi_on_nfs_sr: VDI, defer: Defer):
         full_vdi_write(storage_test_vm, vdi_on_nfs_sr, defer)