Add CBT (Changed Block Tracking) test suite

lib/vdi.py

@@ -23,6 +23,7 @@
 
 ImageFormat = Literal['qcow2', 'raw', 'vhd']
 
+
 class VDI:
     xe_prefix = "vdi"
     sr: SR
@@ -53,6 +54,10 @@ def destroy(self) -> None:
         logging.info("Destroy %s", self)
         self.sr.pool.master.xe('vdi-destroy', {'uuid': self.uuid})
 
+    def data_destroy(self) -> None:
+        logging.info("Data-destroy %s", self)
+        self.sr.pool.master.xe('vdi-data-destroy', {'uuid': self.uuid})
+
     def clone(self) -> VDI:
         uuid = self.sr.pool.master.xe('vdi-clone', {'uuid': self.uuid})
         return VDI(uuid, sr=self.sr)
@@ -124,3 +129,25 @@ def wait_for_coalesce(self, fn: Callable[[], R] | None = None) -> R | None:
         wait_for(lambda: self.get_parent() != previous_parent, msg="Waiting for coalesce", timeout_secs=10 * 60)
         logging.info("Coalesce done")
         return ret
+
+    def enable_cbt(self) -> None:
+        logging.info(f"Enabling CBT on VDI {self.uuid}")
+        self.sr.pool.master.xe('vdi-enable-cbt', {'uuid': self.uuid})
+
+    def disable_cbt(self) -> None:
+        logging.info(f"Disabling CBT on VDI {self.uuid}")
+        self.sr.pool.master.xe('vdi-disable-cbt', {'uuid': self.uuid})
+
+    def get_cbt_enabled(self) -> str:
+        return self.param_get('cbt-enabled')
+
+    def is_cbt_enabled(self) -> bool:
+        return self.get_cbt_enabled() == 'true'
+
+    def list_changed_blocks(self, vdi_to: 'VDI') -> str:
+
+        logging.info(f"Listing changed blocks from VDI {self.uuid} to {vdi_to.uuid}")
+        return self.sr.pool.master.xe('vdi-list-changed-blocks', {
+            'vdi-from-uuid': self.uuid,
+            'vdi-to-uuid': vdi_to.uuid
+        })

tests/storage/__init__.py

@@ -1,16 +1,32 @@
 from .storage import (
     MAX_VDI_SIZE,
+    CBTTest,
     CoalesceOperation,
     ImageFormat,
     XVACompression,
+    assert_cbt_disabled,
+    assert_cbt_enabled,
+    assert_cbt_log_does_not_exist_file_sr,
+    assert_cbt_log_does_not_exist_lvm_sr,
+    assert_cbt_log_exists_file_sr,
+    assert_cbt_log_exists_lvm_sr,
+    assert_changed_blocks_exist,
+    assert_no_changed_blocks,
+    cbt_enabled,
     coalesce_integrity,
     cold_migration_then_come_back,
+    disable_cbt_with_wait,
+    enable_cbt_with_wait,
     full_vdi_write,
+    get_vdi_from_vm,
     install_randstream,
     live_storage_migration_then_come_back,
     randstream,
     try_to_create_sr_with_missing_device,
     vdi_export_import,
     vdi_is_open,
+    verify_changed_blocks_detected,
+    wait_for_cbt_disabled,
+    wait_for_cbt_enabled,
     xva_export_import,
 )

tests/storage/ext/test_ext_sr.py

@@ -13,9 +13,12 @@
 from lib.vm import VM
 from tests.storage import (
     MAX_VDI_SIZE,
+    CBTTest,
     CoalesceOperation,
     ImageFormat,
     XVACompression,
+    assert_cbt_log_does_not_exist_file_sr,
+    assert_cbt_log_exists_file_sr,
     coalesce_integrity,
     full_vdi_write,
     try_to_create_sr_with_missing_device,
@@ -142,3 +145,59 @@ def test_reboot(self, host: Host, ext_sr: SR, vm_on_ext_sr: VM) -> None:
         vm.shutdown(verify=True)
 
     # *** End of tests with reboots
+
+
+class TestEXTCBT(CBTTest):
+    """Test CBT functionality on EXT SR"""
+
+    @staticmethod
+    def assert_cbt_log_exists(host: Host, sr: SR, vdi: VDI) -> None:
+        assert_cbt_log_exists_file_sr(host, sr, vdi)
+
+    @staticmethod
+    def assert_cbt_log_does_not_exist(host: Host, sr: SR, vdi: VDI) -> None:
+        assert_cbt_log_does_not_exist_file_sr(host, sr, vdi)
+
+    def test_enable_disable_cbt(self, host: Host, ext_sr: SR, vdi_on_ext_sr: VDI) -> None:
+        self._test_enable_disable_cbt(host, ext_sr, vdi_on_ext_sr)
+
+    def test_cbt_log_creation(self, host: Host, ext_sr: SR, vdi_on_ext_sr: VDI) -> None:
+        self._test_cbt_log_creation(host, ext_sr, vdi_on_ext_sr)
+
+    def test_snapshot_with_cbt(self, host: Host, ext_sr: SR, vdi_on_ext_sr: VDI) -> None:
+        self._test_snapshot_with_cbt(host, ext_sr, vdi_on_ext_sr)
+
+    @pytest.mark.small_vm
+    def test_changed_blocks_tracking(self, host: Host, ext_sr: SR, vdi_on_ext_sr: VDI, vm_on_ext_sr: VM) -> None:
+        self._test_changed_blocks_tracking(host, ext_sr, vdi_on_ext_sr, vm_on_ext_sr)
+
+    @pytest.mark.small_vm
+    def test_cbt_after_coalesce(self, host: Host, ext_sr: SR, vdi_on_ext_sr: VDI, vm_on_ext_sr: VM) -> None:
+        self._test_cbt_after_coalesce(host, ext_sr, vdi_on_ext_sr, vm_on_ext_sr)
+
+    @pytest.mark.small_vm
+    def test_incremental_snap_scenario(self, host: Host, ext_sr: SR, vdi_on_ext_sr: VDI, vm_on_ext_sr: VM) -> None:
+        self._test_incremental_snap_scenario(host, ext_sr, vdi_on_ext_sr, vm_on_ext_sr)
+
+    def test_disable_cbt_removes_log(self, host: Host, ext_sr: SR, vdi_on_ext_sr: VDI) -> None:
+        self._test_disable_cbt_removes_log(host, ext_sr, vdi_on_ext_sr)
+
+    def test_destroy_vdi_removes_cbt_log(self, host: Host, ext_sr: SR, vdi_on_ext_sr: VDI) -> None:
+        self._test_destroy_vdi_removes_cbt_log(host, ext_sr, vdi_on_ext_sr)
+
+    def test_cbt_persist_after_sr_reboot(self, host: Host, ext_sr: SR, vdi_on_ext_sr: VDI) -> None:
+        self._test_cbt_persist_after_sr_reboot(host, ext_sr, vdi_on_ext_sr)
+
+    def test_cbt_on_snapshot_chain(self, host: Host, ext_sr: SR, vdi_on_ext_sr: VDI) -> None:
+        self._test_cbt_on_snapshot_chain(host, ext_sr, vdi_on_ext_sr)
+
+    def test_cbt_parent_disable_does_not_affect_snapshot(self, host: Host, ext_sr: SR, vdi_on_ext_sr: VDI) -> None:
+        self._test_cbt_parent_disable_does_not_affect_snapshot(host, ext_sr, vdi_on_ext_sr)
+
+    @pytest.mark.small_vm
+    def test_cbt_bitmap_non_zero_after_write(self, host: Host, ext_sr: SR, vdi_on_ext_sr: VDI,
+                                             vm_on_ext_sr: VM) -> None:
+        self._test_cbt_bitmap_non_zero_after_write(host, ext_sr, vdi_on_ext_sr, vm_on_ext_sr)
+
+    def test_cbt_data_destroy(self, host: Host, ext_sr: SR, vdi_on_ext_sr: VDI) -> None:
+        self._test_cbt_data_destroy(host, ext_sr, vdi_on_ext_sr)

tests/storage/lvm/test_lvm_sr.py

@@ -13,9 +13,12 @@
 from lib.vm import VM
 from tests.storage import (
     MAX_VDI_SIZE,
+    CBTTest,
     CoalesceOperation,
     ImageFormat,
     XVACompression,
+    assert_cbt_log_does_not_exist_lvm_sr,
+    assert_cbt_log_exists_lvm_sr,
     coalesce_integrity,
     full_vdi_write,
     try_to_create_sr_with_missing_device,
@@ -161,3 +164,59 @@ def test_reboot(self, host: Host, lvm_sr: SR, vm_on_lvm_sr: VM) -> None:
         vm.shutdown(verify=True)
 
     # *** End of tests with reboots
+
+
+class TestLVMCBT(CBTTest):
+    """Test CBT functionality on LVM SR"""
+
+    @staticmethod
+    def assert_cbt_log_exists(host: Host, sr: SR, vdi: VDI) -> None:
+        assert_cbt_log_exists_lvm_sr(host, sr, vdi)
+
+    @staticmethod
+    def assert_cbt_log_does_not_exist(host: Host, sr: SR, vdi: VDI) -> None:
+        assert_cbt_log_does_not_exist_lvm_sr(host, sr, vdi)
+
+    def test_enable_disable_cbt(self, host: Host, lvm_sr: SR, vdi_on_lvm_sr: VDI) -> None:
+        self._test_enable_disable_cbt(host, lvm_sr, vdi_on_lvm_sr)
+
+    def test_cbt_log_creation(self, host: Host, lvm_sr: SR, vdi_on_lvm_sr: VDI) -> None:
+        self._test_cbt_log_creation(host, lvm_sr, vdi_on_lvm_sr)
+
+    def test_snapshot_with_cbt(self, host: Host, lvm_sr: SR, vdi_on_lvm_sr: VDI) -> None:
+        self._test_snapshot_with_cbt(host, lvm_sr, vdi_on_lvm_sr)
+
+    @pytest.mark.small_vm
+    def test_changed_blocks_tracking(self, host: Host, lvm_sr: SR, vdi_on_lvm_sr: VDI, vm_on_lvm_sr: VM) -> None:
+        self._test_changed_blocks_tracking(host, lvm_sr, vdi_on_lvm_sr, vm_on_lvm_sr)
+
+    @pytest.mark.small_vm
+    def test_cbt_after_coalesce(self, host: Host, lvm_sr: SR, vdi_on_lvm_sr: VDI, vm_on_lvm_sr: VM) -> None:
+        self._test_cbt_after_coalesce(host, lvm_sr, vdi_on_lvm_sr, vm_on_lvm_sr)
+
+    @pytest.mark.small_vm
+    def test_incremental_snap_scenario(self, host: Host, lvm_sr: SR, vdi_on_lvm_sr: VDI, vm_on_lvm_sr: VM) -> None:
+        self._test_incremental_snap_scenario(host, lvm_sr, vdi_on_lvm_sr, vm_on_lvm_sr)
+
+    def test_disable_cbt_removes_log(self, host: Host, lvm_sr: SR, vdi_on_lvm_sr: VDI) -> None:
+        self._test_disable_cbt_removes_log(host, lvm_sr, vdi_on_lvm_sr)
+
+    def test_destroy_vdi_removes_cbt_log(self, host: Host, lvm_sr: SR, vdi_on_lvm_sr: VDI) -> None:
+        self._test_destroy_vdi_removes_cbt_log(host, lvm_sr, vdi_on_lvm_sr)
+
+    def test_cbt_persist_after_sr_reboot(self, host: Host, lvm_sr: SR, vdi_on_lvm_sr: VDI) -> None:
+        self._test_cbt_persist_after_sr_reboot(host, lvm_sr, vdi_on_lvm_sr)
+
+    def test_cbt_on_snapshot_chain(self, host: Host, lvm_sr: SR, vdi_on_lvm_sr: VDI) -> None:
+        self._test_cbt_on_snapshot_chain(host, lvm_sr, vdi_on_lvm_sr)
+
+    def test_cbt_parent_disable_does_not_affect_snapshot(self, host: Host, lvm_sr: SR, vdi_on_lvm_sr: VDI) -> None:
+        self._test_cbt_parent_disable_does_not_affect_snapshot(host, lvm_sr, vdi_on_lvm_sr)
+
+    @pytest.mark.small_vm
+    def test_cbt_bitmap_non_zero_after_write(self, host: Host, lvm_sr: SR, vdi_on_lvm_sr: VDI,
+                                             vm_on_lvm_sr: VM) -> None:
+        self._test_cbt_bitmap_non_zero_after_write(host, lvm_sr, vdi_on_lvm_sr, vm_on_lvm_sr)
+
+    def test_cbt_data_destroy(self, host: Host, lvm_sr: SR, vdi_on_lvm_sr: VDI) -> None:
+        self._test_cbt_data_destroy(host, lvm_sr, vdi_on_lvm_sr)

tests/storage/lvmohba/test_lvmohba_sr.py

@@ -2,14 +2,18 @@
 
 from lib.commands import SSHCommandFailed
 from lib.common import Defer, vm_image, wait_for
+from lib.host import Host
 from lib.sr import SR
-from lib.vdi import VDI, ImageFormat
+from lib.vdi import VDI
 from lib.vm import VM
 from tests.storage import (
     MAX_VDI_SIZE,
+    CBTTest,
     CoalesceOperation,
     ImageFormat,
     XVACompression,
+    assert_cbt_log_does_not_exist_lvm_sr,
+    assert_cbt_log_exists_lvm_sr,
     coalesce_integrity,
     full_vdi_write,
     vdi_export_import,
@@ -93,3 +97,63 @@ def test_reboot(self, host, lvmohba_sr, vm_on_lvmohba_sr):
         vm.shutdown(verify=True)
 
     # *** End of tests with reboots
+
+
+class TestLVMoHBACBT(CBTTest):
+    """Test CBT functionality on LVMOHBA SR"""
+
+    @staticmethod
+    def assert_cbt_log_exists(host: Host, sr: SR, vdi: VDI) -> None:
+        assert_cbt_log_exists_lvm_sr(host, sr, vdi)
+
+    @staticmethod
+    def assert_cbt_log_does_not_exist(host: Host, sr: SR, vdi: VDI) -> None:
+        assert_cbt_log_does_not_exist_lvm_sr(host, sr, vdi)
+
+    def test_enable_disable_cbt(self, host: Host, lvmohba_sr: SR, vdi_on_lvmohba_sr: VDI) -> None:
+        self._test_enable_disable_cbt(host, lvmohba_sr, vdi_on_lvmohba_sr)
+
+    def test_cbt_log_creation(self, host: Host, lvmohba_sr: SR, vdi_on_lvmohba_sr: VDI) -> None:
+        self._test_cbt_log_creation(host, lvmohba_sr, vdi_on_lvmohba_sr)
+
+    def test_snapshot_with_cbt(self, host: Host, lvmohba_sr: SR, vdi_on_lvmohba_sr: VDI) -> None:
+        self._test_snapshot_with_cbt(host, lvmohba_sr, vdi_on_lvmohba_sr)
+
+    @pytest.mark.small_vm
+    def test_changed_blocks_tracking(self, host: Host, lvmohba_sr: SR, vdi_on_lvmohba_sr: VDI,
+                                     vm_on_lvmohba_sr: VM) -> None:
+        self._test_changed_blocks_tracking(host, lvmohba_sr, vdi_on_lvmohba_sr, vm_on_lvmohba_sr)
+
+    @pytest.mark.small_vm
+    def test_cbt_after_coalesce(self, host: Host, lvmohba_sr: SR, vdi_on_lvmohba_sr: VDI,
+                                vm_on_lvmohba_sr: VM) -> None:
+        self._test_cbt_after_coalesce(host, lvmohba_sr, vdi_on_lvmohba_sr, vm_on_lvmohba_sr)
+
+    @pytest.mark.small_vm
+    def test_incremental_snap_scenario(self, host: Host, lvmohba_sr: SR, vdi_on_lvmohba_sr: VDI,
+                                       vm_on_lvmohba_sr: VM) -> None:
+        self._test_incremental_snap_scenario(host, lvmohba_sr, vdi_on_lvmohba_sr, vm_on_lvmohba_sr)
+
+    def test_disable_cbt_removes_log(self, host: Host, lvmohba_sr: SR, vdi_on_lvmohba_sr: VDI) -> None:
+        self._test_disable_cbt_removes_log(host, lvmohba_sr, vdi_on_lvmohba_sr)
+
+    def test_destroy_vdi_removes_cbt_log(self, host: Host, lvmohba_sr: SR, vdi_on_lvmohba_sr: VDI) -> None:
+        self._test_destroy_vdi_removes_cbt_log(host, lvmohba_sr, vdi_on_lvmohba_sr)
+
+    def test_cbt_persist_after_sr_reboot(self, host: Host, lvmohba_sr: SR, vdi_on_lvmohba_sr: VDI) -> None:
+        self._test_cbt_persist_after_sr_reboot(host, lvmohba_sr, vdi_on_lvmohba_sr)
+
+    def test_cbt_on_snapshot_chain(self, host: Host, lvmohba_sr: SR, vdi_on_lvmohba_sr: VDI) -> None:
+        self._test_cbt_on_snapshot_chain(host, lvmohba_sr, vdi_on_lvmohba_sr)
+
+    def test_cbt_parent_disable_does_not_affect_snapshot(self, host: Host, lvmohba_sr: SR,
+                                                         vdi_on_lvmohba_sr: VDI) -> None:
+        self._test_cbt_parent_disable_does_not_affect_snapshot(host, lvmohba_sr, vdi_on_lvmohba_sr)
+
+    @pytest.mark.small_vm
+    def test_cbt_bitmap_non_zero_after_write(self, host: Host, lvmohba_sr: SR, vdi_on_lvmohba_sr: VDI,
+                                             vm_on_lvmohba_sr: VM) -> None:
+        self._test_cbt_bitmap_non_zero_after_write(host, lvmohba_sr, vdi_on_lvmohba_sr, vm_on_lvmohba_sr)
+
+    def test_cbt_data_destroy(self, host: Host, lvmohba_sr: SR, vdi_on_lvmohba_sr: VDI) -> None:
+        self._test_cbt_data_destroy(host, lvmohba_sr, vdi_on_lvmohba_sr)

tests/storage/lvmoiscsi/test_lvmoiscsi_sr.py

@@ -4,13 +4,16 @@
 from lib.common import Defer, vm_image, wait_for
 from lib.host import Host
 from lib.sr import SR
-from lib.vdi import VDI, ImageFormat
+from lib.vdi import VDI
 from lib.vm import VM
 from tests.storage import (
     MAX_VDI_SIZE,
+    CBTTest,
     CoalesceOperation,
     ImageFormat,
     XVACompression,
+    assert_cbt_log_does_not_exist_lvm_sr,
+    assert_cbt_log_exists_lvm_sr,
     coalesce_integrity,
     full_vdi_write,
     vdi_export_import,
@@ -95,3 +98,63 @@ def test_reboot(self, host: Host, lvmoiscsi_sr: SR, vm_on_lvmoiscsi_sr: VM) -> N
         vm.shutdown(verify=True)
 
     # *** End of tests with reboots
+
+
+class TestLVMoISCSICBT(CBTTest):
+    """Test CBT functionality on LVMOISCSI SR"""
+
+    @staticmethod
+    def assert_cbt_log_exists(host: Host, sr: SR, vdi: VDI) -> None:
+        assert_cbt_log_exists_lvm_sr(host, sr, vdi)
+
+    @staticmethod
+    def assert_cbt_log_does_not_exist(host: Host, sr: SR, vdi: VDI) -> None:
+        assert_cbt_log_does_not_exist_lvm_sr(host, sr, vdi)
+
+    def test_enable_disable_cbt(self, host: Host, lvmoiscsi_sr: SR, vdi_on_lvmoiscsi_sr: VDI) -> None:
+        self._test_enable_disable_cbt(host, lvmoiscsi_sr, vdi_on_lvmoiscsi_sr)
+
+    def test_cbt_log_creation(self, host: Host, lvmoiscsi_sr: SR, vdi_on_lvmoiscsi_sr: VDI) -> None:
+        self._test_cbt_log_creation(host, lvmoiscsi_sr, vdi_on_lvmoiscsi_sr)
+
+    def test_snapshot_with_cbt(self, host: Host, lvmoiscsi_sr: SR, vdi_on_lvmoiscsi_sr: VDI) -> None:
+        self._test_snapshot_with_cbt(host, lvmoiscsi_sr, vdi_on_lvmoiscsi_sr)
+
+    @pytest.mark.small_vm
+    def test_changed_blocks_tracking(self, host: Host, lvmoiscsi_sr: SR, vdi_on_lvmoiscsi_sr: VDI,
+                                     vm_on_lvmoiscsi_sr: VM) -> None:
+        self._test_changed_blocks_tracking(host, lvmoiscsi_sr, vdi_on_lvmoiscsi_sr, vm_on_lvmoiscsi_sr)
+
+    @pytest.mark.small_vm
+    def test_cbt_after_coalesce(self, host: Host, lvmoiscsi_sr: SR, vdi_on_lvmoiscsi_sr: VDI,
+                                vm_on_lvmoiscsi_sr: VM) -> None:
+        self._test_cbt_after_coalesce(host, lvmoiscsi_sr, vdi_on_lvmoiscsi_sr, vm_on_lvmoiscsi_sr)
+
+    @pytest.mark.small_vm
+    def test_incremental_snap_scenario(self, host: Host, lvmoiscsi_sr: SR, vdi_on_lvmoiscsi_sr: VDI,
+                                       vm_on_lvmoiscsi_sr: VM) -> None:
+        self._test_incremental_snap_scenario(host, lvmoiscsi_sr, vdi_on_lvmoiscsi_sr, vm_on_lvmoiscsi_sr)
+
+    def test_disable_cbt_removes_log(self, host: Host, lvmoiscsi_sr: SR, vdi_on_lvmoiscsi_sr: VDI) -> None:
+        self._test_disable_cbt_removes_log(host, lvmoiscsi_sr, vdi_on_lvmoiscsi_sr)
+
+    def test_destroy_vdi_removes_cbt_log(self, host: Host, lvmoiscsi_sr: SR, vdi_on_lvmoiscsi_sr: VDI) -> None:
+        self._test_destroy_vdi_removes_cbt_log(host, lvmoiscsi_sr, vdi_on_lvmoiscsi_sr)
+
+    def test_cbt_persist_after_sr_reboot(self, host: Host, lvmoiscsi_sr: SR, vdi_on_lvmoiscsi_sr: VDI) -> None:
+        self._test_cbt_persist_after_sr_reboot(host, lvmoiscsi_sr, vdi_on_lvmoiscsi_sr)
+
+    def test_cbt_on_snapshot_chain(self, host: Host, lvmoiscsi_sr: SR, vdi_on_lvmoiscsi_sr: VDI) -> None:
+        self._test_cbt_on_snapshot_chain(host, lvmoiscsi_sr, vdi_on_lvmoiscsi_sr)
+
+    def test_cbt_parent_disable_does_not_affect_snapshot(self, host: Host, lvmoiscsi_sr: SR,
+                                                         vdi_on_lvmoiscsi_sr: VDI) -> None:
+        self._test_cbt_parent_disable_does_not_affect_snapshot(host, lvmoiscsi_sr, vdi_on_lvmoiscsi_sr)
+
+    @pytest.mark.small_vm
+    def test_cbt_bitmap_non_zero_after_write(self, host: Host, lvmoiscsi_sr: SR, vdi_on_lvmoiscsi_sr: VDI,
+                                             vm_on_lvmoiscsi_sr: VM) -> None:
+        self._test_cbt_bitmap_non_zero_after_write(host, lvmoiscsi_sr, vdi_on_lvmoiscsi_sr, vm_on_lvmoiscsi_sr)
+
+    def test_cbt_data_destroy(self, host: Host, lvmoiscsi_sr: SR, vdi_on_lvmoiscsi_sr: VDI) -> None:
+        self._test_cbt_data_destroy(host, lvmoiscsi_sr, vdi_on_lvmoiscsi_sr)