Merge #1922

1922: fix: encrypted pool expansion workflow r=abhilashshetty04 a=abhilashshetty04

We were rescanning base bdev of the Pool. Base bdev for the encrypted pool is crypto bdev which cant be resized. Now, we get the Aio bdev backing the crypto bdev.

crypto bdev gets notified via event when aio is resized after which it adjusts the blockcnt.



Co-authored-by: Abhilash Shetty <[email protected]>

Author: mayastor-bors

io-engine/src/lvs/lvs_error.rs

@@ -271,10 +271,14 @@ pub enum LvsError {
         source: BsError,
         name: String,
     },
-    #[snafu(display("Failed to extend pool bdev: {name}"))]
+    #[snafu(display("Pool Bdev not extended: {name}"))]
     BdevNotExtended {
         name: String,
     },
+    #[snafu(display("Failed to resize crypto bdev: {name}"))]
+    CryptoBdevNotResized {
+        name: String,
+    },
 }
 
 /// Map CoreError to errno code.
@@ -285,7 +289,7 @@ impl ToErrno for LvsError {
             Self::PoolCreate { source, .. } => source.to_errno(),
             Self::Export { source, .. } => source.to_errno(),
             Self::Destroy { .. } => Errno::ENXIO,
-            Self::Grow { .. } => Errno::ENXIO,
+            Self::Grow { source, .. } => source.to_errno(),
             Self::PoolNotFound { source, .. } => source.to_errno(),
             Self::InvalidBdev { .. } => Errno::ENXIO,
             Self::Invalid { source, .. } => source.to_errno(),
@@ -313,6 +317,7 @@ impl ToErrno for LvsError {
             Self::MaxExpansionParse { .. } => Errno::EINVAL,
             Self::BdevRescanFailed { source, .. } => source.to_errno(),
             Self::BdevNotExtended { .. } => Errno::EOPNOTSUPP,
+            Self::CryptoBdevNotResized { .. } => Errno::EBUSY,
         }
     }
 }

io-engine/src/lvs/lvs_store.rs

@@ -1,21 +1,22 @@
 use core::f64;
 use std::{convert::TryFrom, fmt::Debug, os::raw::c_void, pin::Pin, ptr::NonNull, str::FromStr};
 
+use crate::sleep::mayastor_sleep;
 use byte_unit::Byte;
 use events_api::event::EventAction;
 use futures::channel::oneshot;
 use nix::errno::Errno;
 use pin_utils::core_reexport::fmt::Formatter;
 
 use spdk_rs::libspdk::{
-    bdev_aio_rescan, spdk_bdev_update_bs_blockcnt, spdk_blob_store, spdk_bs_free_cluster_count,
-    spdk_bs_get_cluster_size, spdk_bs_get_max_growable_size, spdk_bs_get_md_len,
-    spdk_bs_get_page_size, spdk_bs_get_used_md, spdk_bs_total_data_cluster_count, spdk_lvol,
-    spdk_lvol_opts, spdk_lvol_opts_init, spdk_lvol_store, spdk_lvs_grow_live,
-    vbdev_get_lvol_store_by_name, vbdev_get_lvol_store_by_uuid, vbdev_get_lvs_bdev_by_lvs,
-    vbdev_lvol_create_with_opts, vbdev_lvs_create, vbdev_lvs_create_with_uuid, vbdev_lvs_destruct,
-    vbdev_lvs_import, vbdev_lvs_unload, LVOL_CLEAR_WITH_NONE, LVOL_CLEAR_WITH_UNMAP,
-    LVS_CLEAR_WITH_NONE,
+    bdev_aio_rescan, bdev_uring_rescan, spdk_bdev_update_bs_blockcnt, spdk_blob_store,
+    spdk_bs_free_cluster_count, spdk_bs_get_cluster_size, spdk_bs_get_max_growable_size,
+    spdk_bs_get_md_len, spdk_bs_get_page_size, spdk_bs_get_used_md,
+    spdk_bs_total_data_cluster_count, spdk_lvol, spdk_lvol_opts, spdk_lvol_opts_init,
+    spdk_lvol_store, spdk_lvs_grow_live, vbdev_get_lvol_store_by_name,
+    vbdev_get_lvol_store_by_uuid, vbdev_get_lvs_bdev_by_lvs, vbdev_lvol_create_with_opts,
+    vbdev_lvs_create, vbdev_lvs_create_with_uuid, vbdev_lvs_destruct, vbdev_lvs_import,
+    vbdev_lvs_unload, LVOL_CLEAR_WITH_NONE, LVOL_CLEAR_WITH_UNMAP, LVS_CLEAR_WITH_NONE,
 };
 use url::Url;
 
@@ -879,32 +880,48 @@ impl Lvs {
     #[tracing::instrument(level = "debug", err)]
     pub async fn grow(&self) -> Result<(), LvsError> {
         info!("{self:?}: growing lvs...");
+        let lvs_name = self.name();
 
-        let cname = self.base_bdev().name().into_cstring();
-        let uri_str = &self.base_bdev().bdev_uri_str().unwrap_or_default();
-        let url = Url::parse(uri_str).map_err(|source| LvsError::InvalidBdev {
+        let disk_bdev = self
+            .base_bdev()
+            .crypto_base_bdev()
+            .map(Bdev::new)
+            .unwrap_or_else(|| self.base_bdev());
+
+        let uri_str = disk_bdev.bdev_uri_str().unwrap_or_default();
+        let url = Url::parse(&uri_str).map_err(|source| LvsError::InvalidBdev {
             source: BdevError::UriParseFailed {
                 source,
                 uri: uri_str.to_string(),
             },
-            name: self.name().to_string(),
+            name: lvs_name.to_string(),
         })?;
-        if url.scheme() != "malloc" {
-            info!(
-                "Attempting to rescan bdev {:?} part of lvs {:?}, uri {:?}",
-                self.base_bdev().name(),
-                self.name(),
-                self.base_bdev().bdev_uri_str().unwrap_or_default()
-            );
 
-            let errno = unsafe { bdev_aio_rescan(cname.as_ptr() as *mut std::os::raw::c_char) };
+        let bdev = disk_bdev.name().into_cstring();
+        info!("Attempting to rescan bdev: {uri_str} part of lvs {lvs_name}");
 
-            if errno != 0 {
-                return Err(LvsError::BdevRescanFailed {
-                    source: BsError::from_i32(errno),
-                    name: self.base_bdev().name().to_string(),
-                });
-            }
+        // Performs a rescan only for uring or aio devices, this is a no-op for other device types.
+        let errno = match url.scheme() {
+            "uring" => unsafe { bdev_uring_rescan(bdev.as_ptr().cast()) },
+            "aio" => unsafe { bdev_aio_rescan(bdev.as_ptr().cast()) },
+            _ => 0,
+        };
+
+        if errno != 0 {
+            return Err(LvsError::BdevRescanFailed {
+                source: BsError::from_i32(errno),
+                name: self.base_bdev().name().to_string(),
+            });
+        }
+
+        if self.encrypted() && !self.crypto_vbdev_resized().await {
+            error!(
+                "crypto bdev {} has not resized",
+                self.base_bdev().name().to_string()
+            );
+            return Err(LvsError::CryptoBdevNotResized {
+                name: self.base_bdev().name().to_string(),
+            });
         }
 
         let capacity_before_grow = self.capacity();
@@ -925,7 +942,7 @@ impl Lvs {
             .expect("callback gone while growing lvs")
             .to_result(|e| LvsError::Grow {
                 source: BsError::from_i32(e),
-                name: self.name().to_string(),
+                name: lvs_name.to_string(),
             })?;
 
         if self.capacity() == capacity_before_grow {
@@ -939,6 +956,31 @@ impl Lvs {
         Ok(())
     }
 
+    /// When the underlying AIO bdev is resized, crypto bdev receives SPDK_BDEV_EVENT_RESIZE
+    /// event. The crypto bdev then adjusts its block count accordingly. To ensure that this resize
+    /// operation completes before proceeding, we wait briefly for the crypto bdev to update.
+    /// This delay gives the crypto bdev time to process the resize event asynchronously.
+    async fn crypto_vbdev_resized(&self) -> bool {
+        for _i in 1..=30 {
+            let base_bdev = self.base_bdev();
+            let disk_bdev = base_bdev
+                .crypto_base_bdev()
+                .map(Bdev::new)
+                .unwrap_or_else(|| self.base_bdev());
+            let disk_bdev_size = disk_bdev.num_blocks() * disk_bdev.block_len() as u64;
+            let crypto_bdev_size = base_bdev.num_blocks() * base_bdev.block_len() as u64;
+            if crypto_bdev_size == disk_bdev_size {
+                return true;
+            } else {
+                let rx = mayastor_sleep(std::time::Duration::from_millis(100));
+                if rx.await.is_err() {
+                    error!("failed to wait for mayastor_sleep");
+                }
+            }
+        }
+        false
+    }
+
     /// return an iterator for enumerating all snapshots that reside on the pool
     pub fn snapshots(&self) -> Option<impl Iterator<Item = LvolSnapshotDescriptor>> {
         if let Some(bdev) = UntypedBdev::bdev_first() {