test: wait for block device I/O readiness in integration tests

Signed-off-by: Carter Li <carter.li@suse.com>

Author: Carter Li

integration/common/core.py

@@ -387,6 +387,7 @@ def restore_with_frontend(url, engine_name, backup):
     client.volume_frontend_start(FRONTEND_TGT_BLOCKDEV)
     v = client.volume_get()
     assert v.frontendState == "up"
+    get_blockdev(v.name)
     return
 
 
@@ -515,6 +516,9 @@ def snapshot_revert_with_frontend(url, engine_name, name):
     client.volume_frontend_shutdown()
     cmd.snapshot_revert(url, name)
     client.volume_frontend_start(FRONTEND_TGT_BLOCKDEV)
+    v = client.volume_get()
+    assert v.frontendState == "up"
+    get_blockdev(v.name)
 
 
 def cleanup_replica_dir(dir=""):
@@ -547,10 +551,13 @@ def open_replica(grpc_client, size=SIZE):
 
 def get_blockdev(volume):
     dev = blockdev(volume)
-    for i in range(10):
-        if not dev.ready():
-            time.sleep(1)
-    assert dev.ready()
+    ready = False
+    for _ in range(RETRY_COUNTS):
+        ready = dev.ready()
+        if ready:
+            break
+        time.sleep(RETRY_INTERVAL)
+    assert ready
     return dev
 
 
@@ -754,6 +761,9 @@ def expand_volume_with_frontend(grpc_controller_client, size):  # NOQA
     grpc_controller_client.volume_expand(size)
     wait_for_volume_expansion(grpc_controller_client, size)
     grpc_controller_client.volume_frontend_start(FRONTEND_TGT_BLOCKDEV)
+    v = grpc_controller_client.volume_get()
+    assert v.frontendState == "up"
+    get_blockdev(v.name)
 
 
 def wait_for_volume_expansion(grpc_controller_client, size):  # NOQA

integration/common/frontend.py

@@ -1,3 +1,4 @@
+import errno
 import os
 from os import path
 import stat
@@ -85,9 +86,22 @@ def writeat(self, offset, data):
         return writeat_direct(self.dev, offset, data)
 
     def ready(self):
-        if not os.path.exists(self.dev):
-            return False
-        mode = os.stat(self.dev).st_mode
-        if not stat.S_ISBLK(mode):
-            return False
+        fd = None
+        try:
+            st = os.stat(self.dev)
+            if not stat.S_ISBLK(st.st_mode):
+                return False
+            # The block device node can exist before the underlying iSCSI
+            # device is able to service I/O. Probe it with a non-blocking
+            # open + small read so we only report ready once it truly works.
+            fd = os.open(self.dev, os.O_RDONLY | os.O_NONBLOCK)
+            os.pread(fd, PAGE_SIZE, 0)
+        except OSError as e:
+            if e.errno in (errno.ENXIO, errno.ENODEV, errno.ENOENT, errno.EIO,
+                           errno.EAGAIN, errno.EWOULDBLOCK, errno.EBUSY):
+                return False
+            raise
+        finally:
+            if fd is not None:
+                os.close(fd)
         return True