Merge pull request #360 from xcp-ng/mlr/629-xostor-drbdadm-verify

Add tests for corruption recovery on LINSTOR SR

Author: stormi

tests/storage/linstor/conftest.py

@@ -3,12 +3,15 @@
 import pytest
 
 import functools
+import json
 import logging
 import os
+from contextlib import contextmanager
 from dataclasses import dataclass
 
 import lib.commands as commands
 from lib import config
+from lib.common import safe_split
 
 try:
     from data import LINSTOR_REDUNDANCY  # type: ignore
@@ -186,9 +189,25 @@ def vdi_on_linstor_sr(linstor_sr: SR) -> Generator[VDI, None, None]:
     yield vdi
     vdi.destroy()
 
+@contextmanager
+def _vm_on_linstor_sr(host: Host, linstor_sr: SR, vm_ref: str) -> Generator[VM]:
+    """
+    Context manager to provide the fixture lifecycle on a VM on a Linstor SR
+    with different scopes without repeating the code.
+    """
+    vm = host.import_vm(vm_ref, sr_uuid=linstor_sr.uuid)
+    try:
+        yield vm
+    finally:
+        logging.info("<< Destroy VM")
+        vm.destroy(verify=True)
+
 @pytest.fixture(scope='module')
 def vm_on_linstor_sr(host: Host, linstor_sr: SR, vm_ref: str) -> Generator[VM, None, None]:
-    vm = host.import_vm(vm_ref, sr_uuid=linstor_sr.uuid)
-    yield vm
-    logging.info("<< Destroy VM")
-    vm.destroy(verify=True)
+    with _vm_on_linstor_sr(host, linstor_sr, vm_ref) as vm:
+        yield vm
+
+@pytest.fixture(scope='function')
+def vm_on_linstor_sr_function(host: Host, linstor_sr: SR, vm_ref: str) -> Generator[VM]:
+    with _vm_on_linstor_sr(host, linstor_sr, vm_ref) as vm:
+        yield vm

tests/storage/linstor/test_linstor_sr.py

@@ -1,23 +1,77 @@
 import pytest
 
+import json
 import logging
+import shlex
 import time
 
 from lib.commands import SSHCommandFailed
-from lib.common import vm_image, wait_for
+from lib.common import safe_split, vm_image, wait_for
 from lib.host import Host
 from lib.pool import Pool
 from lib.sr import SR
 from lib.vdi import VDI
 from lib.vm import VM
 from tests.storage import vdi_is_open
 
-from .conftest import LINSTOR_PACKAGE
+from .conftest import GROUP_NAME, LINSTOR_PACKAGE
+
+from typing import Tuple
 
 # Requirements:
 # - two or more XCP-ng hosts >= 8.2 with additional unused disk(s) for the SR
 # - access to XCP-ng RPM repository from the host
 
+
+def get_drbd_status(host: Host, resource: str):
+    logging.debug("[%s] Fetching DRBD status for resource `%s`...", host, resource)
+    return json.loads(host.ssh(shlex.join(["drbdsetup", "status", resource, "--json"])))
+
+def get_corrupted_resources(host: Host, resource: str):
+    return [
+        (
+            res.get("name", ""),
+            conn.get("name", ""),
+            peer.get("out-of-sync", 0),
+        )
+        for res in get_drbd_status(host, resource)
+        for conn in res.get("connections", [])
+        for peer in conn.get("peer_devices", [])
+        if peer.get("out-of-sync", 0) > 0
+    ]
+
+def wait_drbd_sync(host: Host, resource: str):
+    logging.info("[%s] Waiting for DRBD sync on resource `%s`...", host, resource)
+    host.ssh(shlex.join(["drbdadm", "wait-sync", resource]))
+
+
+def get_vdi_volume_name_from_linstor(master: Host, vdi_uuid: str) -> str:
+    result = master.ssh(shlex.join([
+        "linstor-kv-tool",
+        "--dump-volumes",
+        "-g",
+        f"xcp-sr-{GROUP_NAME}_thin_device"
+    ]))
+    volumes = json.loads(result)
+    for k, v in volumes.items():
+        path = safe_split(k, "/")
+        if len(path) < 4:
+            continue
+        uuid = path[2]
+        data_type = path[3]
+        if uuid == vdi_uuid and data_type == "volume-name":
+            return v
+    raise FileNotFoundError(f"Could not find matching linstor volume for `{vdi_uuid}`")
+
+
+def get_vdi_host(pool: Pool, vdi_uuid: str, path: str) -> Host:
+    for h in pool.hosts:
+        result = h.ssh(shlex.join(["test", "-e", path]), simple_output=False, check=False)
+        if result.returncode == 0:
+            return h
+    raise FileNotFoundError(f"Could not find matching host for `{vdi_uuid}`")
+
+
 @pytest.mark.usefixtures("linstor_sr")
 class TestLinstorSR:
     @pytest.mark.quicktest
@@ -54,6 +108,87 @@ def test_snapshot(self, vm_on_linstor_sr: VM) -> None:
         finally:
             vm.shutdown(verify=True)
 
+    @pytest.fixture(scope='function')
+    def host_and_vm_with_corrupted_vdi_on_linstor_sr(self, host: Host, linstor_sr: SR, vm_on_linstor_sr_function: VM):
+        vm: VM = vm_on_linstor_sr_function
+        pool: Pool = host.pool
+        master: Host = pool.master
+
+        vdi_uuid: str = next((
+            vdi.uuid for vdi in vm.vdis if vdi.sr.uuid == linstor_sr.uuid
+        ))
+
+        volume_name = get_vdi_volume_name_from_linstor(master, vdi_uuid)
+        lv_path = f"/dev/{GROUP_NAME}/{volume_name}_00000"
+        vdi_host = get_vdi_host(pool, vdi_uuid, lv_path)
+        logging.info("[%s]: corrupting `%s`", host, lv_path)
+        vdi_host.ssh(shlex.join([
+            "dd",
+            "if=/dev/urandom",
+            f"of={lv_path}",
+            "bs=4096",
+            # Lower values seem to go undetected sometimes
+            "count=10000"  # ~40MB
+        ]))
+        yield vdi_host, vm, volume_name
+
+    @pytest.mark.small_vm
+    def test_resynchronization(
+        self, host_and_vm_with_corrupted_vdi_on_linstor_sr: Tuple[Host, VM, str]
+    ):
+        (host, vm, resource_name) = host_and_vm_with_corrupted_vdi_on_linstor_sr
+        hostname = host.hostname()
+
+        try:
+            other_host = next(
+                next(h for h in host.pool.hosts if h.hostname() == conn.get("name", ""))
+                for res in get_drbd_status(host, resource_name)
+                for conn in res.get("connections", [])
+                for peer in conn.get("peer_devices", [])
+                if peer.get("peer-disk-state", "") == "UpToDate"
+            )
+            logging.info("Elected `%s` as peer for verification and repair", other_host)
+        except StopIteration:
+            pytest.fail("Could not find an UpToDate peer host")
+
+        corrupted = None
+        max_attempts = 3
+        # Attempting several times since testing revealed `drbdadm verify` can be flaky
+        for attempt in range(1, max_attempts + 1):
+            logging.info("`drbdadm verify` attempt %d/%d", attempt, max_attempts)
+            logging.info("[%s] Running DRBD verify for %s...", other_host, resource_name)
+            other_host.ssh(shlex.join(["drbdadm", "verify", f"{resource_name}:{hostname}/0"]))
+            wait_drbd_sync(other_host, resource_name)
+
+            corrupted_resources = get_corrupted_resources(other_host, resource_name)
+            if not corrupted_resources:
+                logging.warning("No corrupted resources found on attempt #%d", attempt)
+                continue
+            for res_name, peer_name, out_of_sync in corrupted_resources:
+                if res_name == resource_name and peer_name == hostname:
+                    corrupted = (res_name, peer_name, out_of_sync)
+            if corrupted:
+                break
+        if not corrupted:
+            pytest.fail(f"Failed to identify corrupted resource after {max_attempts} attempts")
+
+        logging.info("Invalidating remote resource `%s`...", resource_name)
+        other_host.ssh(shlex.join([
+            "drbdadm", "invalidate-remote",
+            f"{resource_name}:{hostname}/0",
+            "--reset-bitmap=no"
+        ]))
+        wait_drbd_sync(other_host, resource_name)
+        if get_corrupted_resources(other_host, resource_name):
+            pytest.fail("Corrupted resource did not get fixed")
+
+        vm.start(on=host.uuid)
+        try:
+            vm.wait_for_os_booted()
+            vm.test_snapshot_on_running_vm()
+        finally:
+            vm.shutdown(verify=True)
+
     # *** tests with reboots (longer tests).
 
     @pytest.mark.reboot