storage: support for various xva sizes

Signed-off-by: Gaëtan Lehmann <gaetan.lehmann@vates.tech>

Author: glehmann

tests/storage/ext/test_ext_sr.py

@@ -94,8 +94,8 @@ def test_coalesce(
 
     @pytest.mark.small_vm
     @pytest.mark.parametrize("compression", ["none", "gzip", "zstd"])
-    def test_xva_export_import(self, vm_on_ext_sr: VM, compression: XVACompression, defer: Defer):
-        xva_export_import(vm_on_ext_sr, compression, defer)
+    def test_xva_export_import(self, vm_on_ext_sr: VM, compression: XVACompression, temp_large_dir: str, defer: Defer):
+        xva_export_import(vm_on_ext_sr, compression, temp_large_dir, defer)
 
     @pytest.mark.small_vm
     def test_vdi_export_import(

tests/storage/lvm/test_lvm_sr.py

@@ -147,8 +147,8 @@ def test_coalesce(
 
     @pytest.mark.small_vm
     @pytest.mark.parametrize("compression", ["none", "gzip", "zstd"])
-    def test_xva_export_import(self, vm_on_lvm_sr: VM, compression: XVACompression, defer: Defer):
-        xva_export_import(vm_on_lvm_sr, compression, defer)
+    def test_xva_export_import(self, vm_on_lvm_sr: VM, compression: XVACompression, temp_large_dir: str, defer: Defer):
+        xva_export_import(vm_on_lvm_sr, compression, temp_large_dir, defer)
 
     @pytest.mark.small_vm
     def test_vdi_export_import(

tests/storage/lvmoiscsi/test_lvmoiscsi_sr.py

@@ -76,9 +76,9 @@ def test_coalesce(
     @pytest.mark.small_vm
     @pytest.mark.parametrize("compression", ["none", "gzip", "zstd"])
     def test_xva_export_import(
-        self, vm_on_lvmoiscsi_sr: VM, compression: XVACompression, defer: Defer
+        self, vm_on_lvmoiscsi_sr: VM, compression: XVACompression, temp_large_dir: str, defer: Defer
     ):
-        xva_export_import(vm_on_lvmoiscsi_sr, compression, defer)
+        xva_export_import(vm_on_lvmoiscsi_sr, compression, temp_large_dir, defer)
 
     @pytest.mark.small_vm
     def test_vdi_export_import(

tests/storage/nfs/test_nfs_sr.py

@@ -125,8 +125,8 @@ def test_coalesce(
     @pytest.mark.usefixtures('vm_ref')
     @pytest.mark.parametrize('dispatch_nfs', ['vm_on_nfs_sr', 'vm_on_nfs4_sr'], indirect=True)
     @pytest.mark.parametrize("compression", ["none", "gzip", "zstd"])
-    def test_xva_export_import(self, dispatch_nfs: VM, compression: XVACompression, defer: Defer):
-        xva_export_import(dispatch_nfs, compression, defer)
+    def test_xva_export_import(self, dispatch_nfs: VM, compression: XVACompression, temp_large_dir: str, defer: Defer):
+        xva_export_import(dispatch_nfs, compression, temp_large_dir, defer)
 
     @pytest.mark.small_vm
     @pytest.mark.parametrize('dispatch_nfs', ['nfs_sr', 'nfs4_sr'], indirect=True)

tests/storage/storage.py

@@ -219,36 +219,54 @@ def coalesce_integrity(vm: VM, vdi: VDI, vdi_op: CoalesceOperation, defer: Defer
 
 XVACompression = Literal['none', 'gzip', 'zstd']
 
-def xva_export_import(vm: VM, compression: XVACompression, defer: Defer):
-    vm.vdis[0].resize(config.volume_size)
+def xva_export_import(vm: VM, compression: XVACompression, temp_large_dir: str, defer: Defer):
+    # we can't shrink a volume
+    volume_size = max(vm.vdis[0].get_virtual_size(), config.volume_size)
+    vm.vdis[0].resize(volume_size)
     # The tests using this function are using specific fixtures to create the VM on the expected SR
     # In consequence, we can't use the storage_test_vm, so we have to start the VM explicitly and install randstream
     vm.start()
     vm.wait_for_vm_running_and_ssh_up()
     install_randstream(vm)
 
-    # 500MiB, so we have some data to check and some empty spaces in the exported image
-    # TODO: resize the vm partitions and write more data in the disk (like 1/2 of the disk size)
-    #       to ensure we can export and import large xva images
-    randstream(vm, 'generate --size 500MiB /root/data')
-    randstream(vm, 'validate --expected-checksum 24e905d6 /root/data')
+    is_alpine = vm.ssh_with_result('apk --version').returncode == 0
+    if is_alpine:
+        # growpart is not available in alpine 3.12
+        # vm.ssh('apk add cloud-utils-growpart e2fsprogs-extra')
+        vm.ssh('apk add gawk util-linux e2fsprogs-extra')
+        vm.ssh('wget https://raw.githubusercontent.com/canonical/cloud-utils/main/bin/growpart -O /usr/bin/growpart')
+        vm.ssh('chmod +x /usr/bin/growpart')
+        # TODO: maybe use `findmnt -no SOURCE /` from util-linux to get the blockdevice mounted on /
+        growpart_returncode = vm.ssh_with_result('growpart /dev/xvda 3').returncode
+        assert growpart_returncode in [0, 1] # growpart returns 1 if the size is already the expected one
+        vm.ssh('resize2fs /dev/xvda3')
+        stream_size = volume_size // 2
+    else:
+        stream_size = 500 * MiB
+
+    checksum = randstream(vm, f'generate --size {stream_size} /root/data')
+    randstream(vm, f'validate --expected-checksum {checksum} /root/data')
     vm.shutdown(verify=True)
 
-    xva_path = f'/tmp/{vm.uuid}.xva'
+    xva_path = f'{temp_large_dir}/{vm.uuid}.xva'
     defer(lambda: vm.host.ssh(f'rm -f {xva_path}'))
     vm.export(xva_path, compression)
     # check that the zero blocks are not part of the result. Most of the data is from the random stream, so
-    # compression has little effect. We just check the result is between 500 and 700 MiB
+    # compression has little effect. We just take into account the system sie
     size_mb = int(vm.host.ssh(f'du -sm --apparent-size {xva_path}').split()[0])
-    assert 500 < size_mb < 700, f"unexpected xva size: {size_mb}"
+    min_size = stream_size / MiB
+    max_size = stream_size * 1.1 / MiB + 200
+    assert min_size < size_mb < max_size, (
+        f"unexpected xva size {size_mb}MiB, was expected to be between {min_size}MiB and {max_size}MiB"
+    )
 
     imported_vm = vm.host.import_vm(xva_path, vm.vdis[0].sr.uuid)
     defer(lambda: imported_vm.destroy())
-    assert vm.vdis[0].get_virtual_size() == config.volume_size
+    assert vm.vdis[0].get_virtual_size() == volume_size
 
     imported_vm.start()
     imported_vm.wait_for_vm_running_and_ssh_up()
-    randstream(imported_vm, 'validate --expected-checksum 24e905d6 /root/data')
+    randstream(imported_vm, f'validate --expected-checksum {checksum} /root/data')
 
 def vdi_export_import(vm: VM, sr: SR, image_format: ImageFormat, temp_large_dir: str, defer: Defer):
     vdi_src = sr.create_vdi(image_format=image_format, virtual_size=config.volume_size)

tests/storage/xfs/test_xfs_sr.py

@@ -111,8 +111,8 @@ def test_coalesce(
 
     @pytest.mark.small_vm
     @pytest.mark.parametrize("compression", ["none", "gzip", "zstd"])
-    def test_xva_export_import(self, vm_on_xfs_sr: VM, compression: XVACompression, defer: Defer):
-        xva_export_import(vm_on_xfs_sr, compression, defer)
+    def test_xva_export_import(self, vm_on_xfs_sr: VM, compression: XVACompression, temp_large_dir: str, defer: Defer):
+        xva_export_import(vm_on_xfs_sr, compression, temp_large_dir, defer)
 
     @pytest.mark.small_vm
     def test_vdi_export_import(

tests/storage/zfs/test_zfs_sr.py

@@ -113,8 +113,8 @@ def test_coalesce(
 
     @pytest.mark.small_vm
     @pytest.mark.parametrize("compression", ["none", "gzip", "zstd"])
-    def test_xva_export_import(self, vm_on_zfs_sr: VM, compression: XVACompression, defer: Defer):
-        xva_export_import(vm_on_zfs_sr, compression, defer)
+    def test_xva_export_import(self, vm_on_zfs_sr: VM, compression: XVACompression, temp_large_dir: str, defer: Defer):
+        xva_export_import(vm_on_zfs_sr, compression, temp_large_dir, defer)
 
     @pytest.mark.small_vm
     def test_vdi_export_import(

tests/storage/zfsvol/test_zfsvol_sr.py

@@ -79,8 +79,10 @@ def test_coalesce(
 
     @pytest.mark.small_vm
     @pytest.mark.parametrize("compression", ["none", "gzip", "zstd"])
-    def test_xva_export_import(self, vm_on_zfsvol_sr: VM, compression: XVACompression, defer: Defer):
-        xva_export_import(vm_on_zfsvol_sr, compression, defer)
+    def test_xva_export_import(
+        self, vm_on_zfsvol_sr: VM, compression: XVACompression, temp_large_dir: str, defer: Defer
+    ):
+        xva_export_import(vm_on_zfsvol_sr, compression, temp_large_dir, defer)
 
     @pytest.mark.small_vm
     def test_vdi_export_import(