Skip to content

Commit 6f5e3af

Browse files
glehmannglehmann
committed
storage: add jobs for large volume tests
Mark LVM and lvmoiscsi tests that allocate large VDIs with disk_space_allocation, and all test_full_vdi_write tests with disk_throughput_intensive, to allow running large-VDI jobs only on hardware that can handle the respective disk requirements. Add storage-main-16tb-{thin,thick,full-write} and storage-migrations-16tb-{thin,thick} jobs that use these markers to split 16TiB volume testing by disk speed requirement. Signed-off-by: Gaëtan Lehmann <gaetan.lehmann@vates.tech>
1 parent d98d020 commit 6f5e3af

13 files changed

Lines changed: 120 additions & 2 deletions

‎jobs.py‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,62 @@ class JobData(TypedDict):
101101
"markers": "(small_vm or no_vm) and not reboot and not quicktest and not unused_4k_disks",
102102
"name_filter": "not migration and not linstor",
103103
},
104+
"storage-main-16tb-thin": {
105+
"description":
106+
"same as storage-main with 16TiB VDIs, no gzip XVAs and no test requiring 16TiB allocation",
107+
"requirements": [
108+
"A pool with at least 3 hosts.",
109+
"An additional free disk on every host.",
110+
"Configuration in data.py for each remote SR that will be tested.",
111+
"A small VM that can be imported on the SRs.",
112+
],
113+
"nb_pools": 1,
114+
"params": {
115+
"--vm": "single/small_vm",
116+
"--volume-size": "16TiB",
117+
},
118+
"paths": ["tests/storage"],
119+
"markers": "(small_vm or no_vm) and not reboot and not quicktest and not unused_4k_disks"
120+
" and not disk_space_allocation and not disk_throughput_intensive",
121+
"name_filter": "not migration and not linstor and not gzip",
122+
},
123+
"storage-main-16tb-thick": {
124+
"description":
125+
"same as storage-main with 16TiB VDIs, no gzip XVAs and tests requiring 16TiB allocation",
126+
"requirements": [
127+
"A pool with at least 3 hosts.",
128+
"An additional free disk on every host.",
129+
"Configuration in data.py for each remote SR that will be tested.",
130+
"A small VM that can be imported on the SRs.",
131+
],
132+
"nb_pools": 1,
133+
"params": {
134+
"--vm": "single/small_vm",
135+
"--volume-size": "16TiB",
136+
},
137+
"paths": ["tests/storage"],
138+
"markers": "(small_vm or no_vm) and not reboot and not quicktest and not unused_4k_disks"
139+
" and not disk_throughput_intensive and disk_space_allocation",
140+
"name_filter": "not migration and not linstor and not gzip",
141+
},
142+
"storage-main-16tb-full-write": {
143+
"description": "storage tests actually writing the full 16TiB volumes",
144+
"requirements": [
145+
"A pool with at least 3 hosts.",
146+
"An additional free disk on every host.",
147+
"Configuration in data.py for each remote SR that will be tested.",
148+
"A small VM that can be imported on the SRs.",
149+
],
150+
"nb_pools": 1,
151+
"params": {
152+
"--vm": "single/small_vm",
153+
"--volume-size": "16TiB",
154+
},
155+
"paths": ["tests/storage"],
156+
"markers": "(small_vm or no_vm) and not reboot and not quicktest and not unused_4k_disks"
157+
" and disk_throughput_intensive",
158+
"name_filter": "not migration and not linstor and not gzip",
159+
},
104160
"storage-migrations": {
105161
"description": "tests migrations with all storage drivers",
106162
"requirements": [
@@ -118,6 +174,42 @@ class JobData(TypedDict):
118174
"markers": "not unused_4k_disks",
119175
"name_filter": "migration and not linstor",
120176
},
177+
"storage-migrations-16tb-thin": {
178+
"description": "same as storage-migrations with 16TiB VDIs, and no test requiring 16TiB allocation",
179+
"requirements": [
180+
"A pool with at least 3 hosts.",
181+
"An additional free disk on every host.",
182+
"A second pool with at least 1 host and a SR to receive VMs.",
183+
"Configuration in data.py for each remote SR that will be tested.",
184+
"A small VM that can be imported on the SRs.",
185+
],
186+
"nb_pools": 2,
187+
"params": {
188+
"--vm": "single/small_vm",
189+
"--volume-size": "16TiB",
190+
},
191+
"paths": ["tests/storage"],
192+
"markers": "not unused_4k_disks and not disk_space_allocation",
193+
"name_filter": "migration and not linstor",
194+
},
195+
"storage-migrations-16tb-thick": {
196+
"description": "same as storage-migrations with 16TiB VDIs, and tests requiring 16TiB allocation",
197+
"requirements": [
198+
"A pool with at least 3 hosts.",
199+
"An additional free disk on every host.",
200+
"A second pool with at least 1 host and a SR to receive VMs.",
201+
"Configuration in data.py for each remote SR that will be tested.",
202+
"A small VM that can be imported on the SRs.",
203+
],
204+
"nb_pools": 2,
205+
"params": {
206+
"--vm": "single/small_vm",
207+
"--volume-size": "16TiB",
208+
},
209+
"paths": ["tests/storage"],
210+
"markers": "not unused_4k_disks and disk_space_allocation",
211+
"name_filter": "migration and not linstor",
212+
},
121213
"storage-reboots": {
122214
"description": "storage driver tests that involve rebooting hosts (except flaky tests)",
123215
"requirements": [

‎pytest.ini‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,10 @@ markers =
3232
multi_vms: tests that it would be good to run on a variety of VMs (includes `small_vm` but excludes `big_vm`).
3333
debian_uefi_vm: tests that require a Debian UEFI VM
3434

35+
# * Disk-related markers
36+
disk_space_allocation: tests that allocate a large VDI (disk won't be fully written, slow disk is OK).
37+
disk_throughput_intensive: tests that fill a large VDI entirely (require a fast disk).
38+
3539
# * Other markers
3640
reboot: tests that reboot one or more hosts.
3741
flaky: flaky tests. Usually pass, but sometimes fail unexpectedly.

‎tests/storage/ext/test_ext_sr.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ def test_vdi_export_import(self, storage_test_vm: VM, ext_sr: SR, image_format:
104104
vdi_export_import(storage_test_vm, ext_sr, image_format, temp_large_dir, defer)
105105

106106
@pytest.mark.small_vm
107+
@pytest.mark.disk_throughput_intensive
107108
def test_full_vdi_write(self, storage_test_vm: VM, vdi_on_ext_sr: VDI, defer: Defer):
108109
full_vdi_write(storage_test_vm, vdi_on_ext_sr, defer)
109110

‎tests/storage/lvm/test_lvm_sr.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,11 @@ class TestLVMSR:
5959
def test_quicktest(self, lvm_sr: SR) -> None:
6060
lvm_sr.run_quicktest()
6161

62-
def test_vdi_is_not_open(self, vdi_on_lvm_sr: VDI) -> None:
62+
@pytest.mark.disk_space_allocation
63+
def test_vdi_is_not_open(self, vdi_on_lvm_sr) -> None:
6364
assert not vdi_is_open(vdi_on_lvm_sr)
6465

66+
@pytest.mark.disk_space_allocation
6567
def test_vdi_image_format(self, vdi_on_lvm_sr: VDI, image_format: ImageFormat) -> None:
6668
fmt = vdi_on_lvm_sr.get_image_format()
6769
# feature-detect: if the SM doesn't report image-format, skip this check
@@ -145,11 +147,14 @@ def test_failing_resize_on_inflate_after_setSizePhys(
145147
"Inflate journal still exist following the scan"
146148

147149
@pytest.mark.small_vm
150+
@pytest.mark.disk_space_allocation
148151
@pytest.mark.parametrize("vdi_op", ["snapshot", "clone"])
149152
def test_coalesce(self, storage_test_vm: VM, vdi_on_lvm_sr: VDI, vdi_op: CoalesceOperation, defer: Defer) -> None:
150153
coalesce_integrity(storage_test_vm, vdi_on_lvm_sr, vdi_op, defer)
151154

152155
@pytest.mark.small_vm
156+
@pytest.mark.disk_space_allocation
157+
@pytest.mark.disk_throughput_intensive
153158
def test_full_vdi_write(self, storage_test_vm: VM, vdi_on_lvm_sr: VDI, defer: Defer):
154159
full_vdi_write(storage_test_vm, vdi_on_lvm_sr, defer)
155160

@@ -161,11 +166,13 @@ def test_invalid_vdi_size(self, lvm_sr: SR, image_format: ImageFormat):
161166

162167
@pytest.mark.small_vm
163168
@pytest.mark.parametrize("compression", ["none", "gzip", "zstd"])
169+
@pytest.mark.disk_space_allocation
164170
def test_xva_export_import(self, vm_on_lvm_sr: VM, compression: XVACompression, temp_large_dir: str, defer: Defer) \
165171
-> None:
166172
xva_export_import(vm_on_lvm_sr, compression, temp_large_dir, defer)
167173

168174
@pytest.mark.small_vm
175+
@pytest.mark.disk_space_allocation
169176
def test_vdi_export_import(self, storage_test_vm: VM, lvm_sr: SR, image_format: ImageFormat, temp_large_dir: str,
170177
defer: Defer) -> None:
171178
vdi_export_import(storage_test_vm, lvm_sr, image_format, temp_large_dir, defer)

‎tests/storage/lvm/test_lvm_sr_crosspool_migration.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
@pytest.mark.small_vm # run with a small VM to test the features
1616
@pytest.mark.big_vm # and ideally with a big VM to test it scales
1717
@pytest.mark.usefixtures("hostB1", "local_sr_on_hostB1")
18+
@pytest.mark.disk_space_allocation
1819
class Test:
1920
def test_cold_crosspool_migration(self, host: Host, hostB1: Host, vm_on_lvm_sr: VM, local_sr_on_hostB1: SR) -> None:
2021
cold_migration_then_come_back(vm_on_lvm_sr, host, hostB1, local_sr_on_hostB1)

‎tests/storage/lvm/test_lvm_sr_intrapool_migration.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
@pytest.mark.small_vm # run with a small VM to test the features
1616
@pytest.mark.big_vm # and ideally with a big VM to test it scales
1717
@pytest.mark.usefixtures("hostA2", "local_sr_on_hostA2")
18+
@pytest.mark.disk_space_allocation
1819
class Test:
1920
def test_cold_intrapool_migration(self, host: Host, hostA2: Host, vm_on_lvm_sr: VM, local_sr_on_hostA2: SR) -> None:
2021
cold_migration_then_come_back(vm_on_lvm_sr, host, hostA2, local_sr_on_hostA2)

‎tests/storage/lvmoiscsi/test_lvmoiscsi_sr.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,8 @@ class TestLVMOISCSISR:
4848
def test_quicktest(self, lvmoiscsi_sr: SR) -> None:
4949
lvmoiscsi_sr.run_quicktest()
5050

51-
def test_vdi_is_not_open(self, vdi_on_lvmoiscsi_sr: VDI) -> None:
51+
@pytest.mark.disk_space_allocation
52+
def test_vdi_is_not_open(self, vdi_on_lvmoiscsi_sr) -> None:
5253
assert not vdi_is_open(vdi_on_lvmoiscsi_sr)
5354

5455
@pytest.mark.small_vm # run with a small VM to test the features
@@ -72,11 +73,14 @@ def test_snapshot(self, vm_on_lvmoiscsi_sr: VM) -> None:
7273

7374
@pytest.mark.small_vm
7475
@pytest.mark.parametrize("vdi_op", ["snapshot", "clone"])
76+
@pytest.mark.disk_space_allocation
7577
def test_coalesce(self, storage_test_vm: 'VM', vdi_on_lvmoiscsi_sr: 'VDI', vdi_op: CoalesceOperation,
7678
defer: Defer) -> None:
7779
coalesce_integrity(storage_test_vm, vdi_on_lvmoiscsi_sr, vdi_op, defer)
7880

7981
@pytest.mark.small_vm
82+
@pytest.mark.disk_space_allocation
83+
@pytest.mark.disk_throughput_intensive
8084
def test_full_vdi_write(self, storage_test_vm: VM, vdi_on_lvmoiscsi_sr: VDI, defer: Defer):
8185
full_vdi_write(storage_test_vm, vdi_on_lvmoiscsi_sr, defer)
8286

@@ -88,11 +92,13 @@ def test_invalid_vdi_size(self, lvmoiscsi_sr: SR, image_format: ImageFormat):
8892

8993
@pytest.mark.small_vm
9094
@pytest.mark.parametrize("compression", ["none", "gzip", "zstd"])
95+
@pytest.mark.disk_space_allocation
9196
def test_xva_export_import(self, vm_on_lvmoiscsi_sr: VM, compression: XVACompression, temp_large_dir: str,
9297
defer: Defer) -> None:
9398
xva_export_import(vm_on_lvmoiscsi_sr, compression, temp_large_dir, defer)
9499

95100
@pytest.mark.small_vm
101+
@pytest.mark.disk_space_allocation
96102
def test_vdi_export_import(self, storage_test_vm: VM, lvmoiscsi_sr: SR, image_format: ImageFormat,
97103
temp_large_dir: str, defer: Defer) -> None:
98104
vdi_export_import(storage_test_vm, lvmoiscsi_sr, image_format, temp_large_dir, defer)

‎tests/storage/lvmoiscsi/test_lvmoiscsi_sr_crosspool_migration.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
@pytest.mark.small_vm # run with a small VM to test the features
1616
@pytest.mark.big_vm # and ideally with a big VM to test it scales
1717
@pytest.mark.usefixtures("hostB1", "local_sr_on_hostB1")
18+
@pytest.mark.disk_space_allocation
1819
class Test:
1920
def test_cold_crosspool_migration(
2021
self, host: Host, hostB1: Host, vm_on_lvmoiscsi_sr: VM, local_sr_on_hostB1: SR

‎tests/storage/lvmoiscsi/test_lvmoiscsi_sr_intrapool_migration.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
@pytest.mark.small_vm # run with a small VM to test the features
1616
@pytest.mark.big_vm # and ideally with a big VM to test it scales
1717
@pytest.mark.usefixtures("hostA2", "local_sr_on_hostA2")
18+
@pytest.mark.disk_space_allocation
1819
class Test:
1920
def test_live_intrapool_shared_migration(self, host: Host, hostA2: Host, vm_on_lvmoiscsi_sr: VM) -> None:
2021
sr = vm_on_lvmoiscsi_sr.get_sr()

‎tests/storage/nfs/test_nfs_sr.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ def test_coalesce(self, storage_test_vm: VM, dispatch_nfs: VDI, vdi_op: Coalesce
122122
coalesce_integrity(storage_test_vm, dispatch_nfs, vdi_op, defer)
123123

124124
@pytest.mark.small_vm
125+
@pytest.mark.disk_throughput_intensive
125126
def test_full_vdi_write(self, storage_test_vm: VM, vdi_on_nfs_sr: VDI, defer: Defer):
126127
full_vdi_write(storage_test_vm, vdi_on_nfs_sr, defer)
127128

0 commit comments

Comments
 (0)