Try #1923:

Author: mayastor-bors

io-engine-tests/src/error_bdev.rs

@@ -13,7 +13,14 @@ pub fn create_error_bdev(error_device: &str, backing_device: &str) {
 
     unsafe {
         // this allows us to create a bdev without its name being a uri
-        retval = create_aio_bdev(cname.as_ptr(), filename.as_ptr(), 512, false, false)
+        retval = create_aio_bdev(
+            cname.as_ptr(),
+            filename.as_ptr(),
+            512,
+            false,
+            false,
+            std::ptr::null_mut(),
+        )
     };
     assert_eq!(retval, 0);
 

io-engine/src/bdev/aio.rs

@@ -116,8 +116,16 @@ impl CreateDestroy for Aio {
 
         let cname = CString::new(self.get_name()).unwrap();
 
-        let errno =
-            unsafe { create_aio_bdev(cname.as_ptr(), cname.as_ptr(), self.blk_size, false, false) };
+        let errno = unsafe {
+            create_aio_bdev(
+                cname.as_ptr(),
+                cname.as_ptr(),
+                self.blk_size,
+                false,
+                false,
+                std::ptr::null_mut(),
+            )
+        };
 
         if errno != 0 {
             let err = BdevError::CreateBdevFailed {

io-engine/src/bdev/malloc.rs

@@ -203,6 +203,7 @@ impl CreateDestroy for Malloc {
                 md_interleave: false,
                 dif_type: SPDK_DIF_DISABLE,
                 dif_is_head_of_md: false,
+                dif_pi_format: 0,
             };
 
             create_malloc_disk(&mut bdev, &opts)

io-engine/src/bdev/nexus/nexus_bdev.rs

@@ -1339,9 +1339,9 @@ impl<'n> BdevOps for Nexus<'n> {
 
     fn io_type_supported(&self, io_type: IoType) -> bool {
         match io_type {
+            IoType::NvmeAdmin => false,
             // we always assume the device supports read/write commands
-            // allow NVMe Admin as it is needed for local replicas
-            IoType::Read | IoType::Write | IoType::NvmeAdmin => true,
+            IoType::Read | IoType::Write => true,
             IoType::Flush | IoType::Reset | IoType::Unmap | IoType::WriteZeros => {
                 let supported = self.io_is_supported(io_type);
                 if !supported {

io-engine/src/bdev/null_bdev.rs

@@ -156,16 +156,16 @@ impl CreateDestroy for Null {
         let uuid = spdk_rs::Uuid::generate().into_raw();
 
         let cname = self.name.clone().into_cstring();
-        let opts = spdk_rs::libspdk::spdk_null_bdev_opts {
-            name: cname.as_ptr(),
-            uuid: &uuid,
+        let opts = spdk_rs::libspdk::null_bdev_opts {
+            name: cname.into_raw(),
+            uuid,
             num_blocks: self.num_blocks,
             block_size: self.blk_size,
             physical_block_size: 0,
             md_size: 0,
-            md_interleave: false,
             dif_type: spdk_rs::libspdk::SPDK_DIF_DISABLE,
             dif_is_head_of_md: false,
+            dif_pi_format: 0,
         };
 
         let errno = unsafe {

io-engine/src/bdev/nvme.rs

@@ -12,7 +12,7 @@ use url::Url;
 use spdk_rs::{
     bdevs::bdev_nvme_delete_async,
     ffihelper::copy_str_with_null,
-    libspdk::{bdev_nvme_create, spdk_nvme_transport_id},
+    libspdk::{spdk_bdev_nvme_create, spdk_nvme_transport_id},
 };
 
 use crate::{
@@ -80,7 +80,7 @@ impl CreateDestroy for NVMe {
         let (sender, receiver) = oneshot::channel::<ErrnoResult<()>>();
 
         let errno = unsafe {
-            bdev_nvme_create(
+            spdk_bdev_nvme_create(
                 &mut context.trid,
                 cname.as_ptr(),
                 &mut context.names[0],
@@ -89,7 +89,6 @@ impl CreateDestroy for NVMe {
                 cb_arg(sender),
                 std::ptr::null_mut(),
                 std::ptr::null_mut(),
-                false,
             )
         };
 

io-engine/src/bdev/nvmf.rs

@@ -14,7 +14,7 @@ use spdk_rs::{
     bdevs::bdev_nvme_delete_async,
     ffihelper::copy_str_with_null,
     libspdk::{
-        bdev_nvme_create, spdk_nvme_transport_id, SPDK_NVME_IO_FLAGS_PRCHK_GUARD,
+        spdk_bdev_nvme_create, spdk_nvme_transport_id, SPDK_NVME_IO_FLAGS_PRCHK_GUARD,
         SPDK_NVME_IO_FLAGS_PRCHK_REFTAG, SPDK_NVME_TRANSPORT_TCP, SPDK_NVMF_ADRFAM_IPV4,
     },
 };
@@ -150,7 +150,7 @@ impl CreateDestroy for Nvmf {
         let (sender, receiver) = oneshot::channel::<ErrnoResult<usize>>();
 
         let errno = unsafe {
-            bdev_nvme_create(
+            spdk_bdev_nvme_create(
                 &mut context.trid,
                 cname.as_ptr(),
                 &mut context.names[0],
@@ -159,7 +159,6 @@ impl CreateDestroy for Nvmf {
                 cb_arg(sender),
                 std::ptr::null_mut(), // context.prchk_flags,
                 std::ptr::null_mut(),
-                false,
             )
         };
 

io-engine/src/bdev/nvmx/controller_inner.rs

@@ -363,11 +363,16 @@ impl NvmeController<'_> {
         // Check Controller Fatal Status for non-admin commands only to avoid
         // endless command resubmission in case of disconnected qpair.
         if !qpair.is_null()
-            && spdk_ctrlr.check_cfs()
+            // Commenting out CFS check for now as it blocks when there is a total communication loss
+            // to the controller, where admin commands time out as well.
+            // We started taking this path of IO qpair timeout after upstream commit:
+            // https://github.com/spdk/spdk/commit/8f9167af6f87b3c95cbc32798c47129e382e981f
+            //&& spdk_ctrlr.check_cfs()
+            && timeout_action != DeviceTimeoutAction::Ignore
             && timeout_action != DeviceTimeoutAction::HotRemove
         {
             error!(
-                "{}: controller Fatal Status set, reset required",
+                "{}: controller IO qpair timeout, reset required",
                 timeout_cfg.name
             );
             timeout_action = DeviceTimeoutAction::Reset;

io-engine/src/lvs/lvs_store.rs

@@ -15,7 +15,7 @@ use spdk_rs::libspdk::{
     spdk_bs_total_data_cluster_count, spdk_lvol, spdk_lvol_opts, spdk_lvol_opts_init,
     spdk_lvol_store, spdk_lvs_grow_live, vbdev_get_lvol_store_by_name,
     vbdev_get_lvol_store_by_uuid, vbdev_get_lvs_bdev_by_lvs, vbdev_lvol_create_with_opts,
-    vbdev_lvs_create, vbdev_lvs_create_with_uuid, vbdev_lvs_destruct, vbdev_lvs_import,
+    vbdev_lvs_create_ext, vbdev_lvs_create_with_uuid, vbdev_lvs_destruct, vbdev_lvs_import,
     vbdev_lvs_unload, LVOL_CLEAR_WITH_NONE, LVOL_CLEAR_WITH_UNMAP, LVS_CLEAR_WITH_NONE,
 };
 use url::Url;
@@ -517,11 +517,12 @@ impl Lvs {
                     // acceptable.
                     LVS_CLEAR_WITH_NONE,
                     mdp_ratio,
+                    0,
                     Some(Self::lvs_cb),
                     cb_arg(sender),
                 )
             } else {
-                vbdev_lvs_create(
+                vbdev_lvs_create_ext(
                     bdev_name.as_ptr(),
                     pool_name.as_ptr(),
                     cluster_size,
@@ -533,6 +534,7 @@ impl Lvs {
                     // acceptable.
                     LVS_CLEAR_WITH_NONE,
                     mdp_ratio,
+                    0,
                     Some(Self::lvs_cb),
                     cb_arg(sender),
                 )

io-engine/src/subsys/config/opts.rs

@@ -8,10 +8,11 @@ use serde::{Deserialize, Serialize};
 use spdk_rs::{
     ffihelper::copy_str_with_null,
     libspdk::{
-        bdev_nvme_get_opts, bdev_nvme_set_opts, spdk_bdev_get_opts, spdk_bdev_nvme_opts,
+        spdk_bdev_get_opts, spdk_bdev_nvme_get_opts, spdk_bdev_nvme_opts, spdk_bdev_nvme_set_opts,
         spdk_bdev_opts, spdk_bdev_set_opts, spdk_iobuf_get_opts, spdk_iobuf_opts,
         spdk_iobuf_set_opts, spdk_nvmf_target_opts, spdk_nvmf_transport_opts,
-        spdk_sock_impl_get_opts, spdk_sock_impl_opts, spdk_sock_impl_set_opts,
+        spdk_sock_impl_get_opts, spdk_sock_impl_opts, spdk_sock_impl_set_opts, NVMF_DEFAULT_KAS,
+        NVMF_DEFAULT_MIN_KATO,
     },
     struct_size_init,
 };
@@ -314,7 +315,9 @@ impl From<NvmfTransportOpts> for spdk_nvmf_transport_opts {
                 num_shared_buffers: o.num_shared_buf,
                 buf_cache_size: o.buf_cache_size,
                 dif_insert_or_strip: o.dif_insert_or_strip,
-                reserved29: Default::default(),
+                // todo: add disable_command_passthru in NvmfTransportOpts when needed.
+                disable_command_passthru: false,
+                reserved30: Default::default(),
                 abort_timeout_sec: o.abort_timeout_sec,
                 association_timeout: 120000,
                 transport_specific: std::ptr::null(),
@@ -323,6 +326,9 @@ impl From<NvmfTransportOpts> for spdk_nvmf_transport_opts {
                 reserved61: Default::default(),
                 ack_timeout: o.ack_timeout,
                 data_wr_pool_size: o.data_wr_pool_size,
+                // todo: add min_kato and kas in NvmfTransportOpts when needed.
+                min_kato: NVMF_DEFAULT_MIN_KATO,
+                kas: NVMF_DEFAULT_KAS as u16,
             },
             opts_size
         )
@@ -381,14 +387,19 @@ pub struct NvmeBdevOpts {
 impl GetOpts for NvmeBdevOpts {
     fn get(&self) -> Self {
         let opts: spdk_bdev_nvme_opts = unsafe { zeroed() };
-        unsafe { bdev_nvme_get_opts(&opts as *const _ as *mut spdk_bdev_nvme_opts) };
+        unsafe {
+            spdk_bdev_nvme_get_opts(
+                &opts as *const _ as *mut spdk_bdev_nvme_opts,
+                size_of::<spdk_bdev_nvme_opts>() as u64,
+            )
+        };
         opts.into()
     }
 
     fn set(&self) -> bool {
         let opts = Box::new(self.into());
         debug!("{:?}", &opts);
-        if unsafe { bdev_nvme_set_opts(Box::into_raw(opts)) } != 0 {
+        if unsafe { spdk_bdev_nvme_set_opts(Box::into_raw(opts)) } != 0 {
             warn!("Failed to apply NVMe Bdev options");
             return false;
         }
@@ -466,6 +477,7 @@ impl From<spdk_bdev_nvme_opts> for NvmeBdevOpts {
 impl From<&NvmeBdevOpts> for spdk_bdev_nvme_opts {
     fn from(o: &NvmeBdevOpts) -> Self {
         Self {
+            opts_size: std::mem::size_of::<spdk_bdev_nvme_opts>() as u64,
             action_on_timeout: o.action_on_timeout,
             timeout_us: o.timeout_us,
             timeout_admin_us: o.timeout_admin_us,
@@ -479,6 +491,7 @@ impl From<&NvmeBdevOpts> for spdk_bdev_nvme_opts {
             nvme_ioq_poll_period_us: o.nvme_ioq_poll_period_us,
             io_queue_requests: o.io_queue_requests,
             delay_cmd_submit: o.delay_cmd_submit,
+            reserved73: Default::default(),
             bdev_retry_count: o.bdev_retry_count,
             transport_ack_timeout: o.transport_ack_timeout,
             ctrlr_loss_timeout_sec: o.ctrlr_loss_timeout_sec,
@@ -490,11 +503,16 @@ impl From<&NvmeBdevOpts> for spdk_bdev_nvme_opts {
             nvme_error_stat: false,
             rdma_srq_size: 0,
             io_path_stat: false,
+            reserved99: Default::default(),
             allow_accel_sequence: false,
             rdma_max_cq_size: 0,
             rdma_cm_event_timeout_ms: 0,
+            reserved110: Default::default(),
             dhchap_digests: 0,
             dhchap_dhgroups: 0,
+            rdma_umr_per_io: false,
+            reserved121: Default::default(),
+            tcp_connect_timeout_ms: 0,
         }
     }
 }
@@ -708,6 +726,8 @@ pub struct IoBufOpts {
     pub small_bufsize: u32,
     /// Size of a single large buffer.
     pub large_bufsize: u32,
+    /// Enable per-NUMA node buffer pools.
+    pub enable_numa: bool,
 }
 
 impl GetOpts for IoBufOpts {
@@ -718,6 +738,7 @@ impl GetOpts for IoBufOpts {
                 large_pool_count: 0,
                 small_bufsize: 0,
                 large_bufsize: 0,
+                enable_numa: 0,
             },
             opts_size
         );
@@ -742,6 +763,7 @@ impl Default for IoBufOpts {
             large_pool_count: try_from_env("IOBUF_LARGE_POOL_COUNT", 2048),
             small_bufsize: try_from_env("IOBUF_SMALL_BUFSIZE", 8 * 1024),
             large_bufsize: try_from_env("IOBUF_LARGE_BUFSIZE", 132 * 1024),
+            enable_numa: try_from_env("IOBUF_ENABLE_NUMA", false),
         }
     }
 }
@@ -753,6 +775,7 @@ impl From<spdk_iobuf_opts> for IoBufOpts {
             large_pool_count: o.large_pool_count,
             small_bufsize: o.small_bufsize,
             large_bufsize: o.large_bufsize,
+            enable_numa: o.enable_numa != 0,
         }
     }
 }
@@ -765,6 +788,7 @@ impl From<&IoBufOpts> for spdk_iobuf_opts {
                 large_pool_count: o.large_pool_count,
                 small_bufsize: o.small_bufsize,
                 large_bufsize: o.large_bufsize,
+                enable_numa: o.enable_numa as u8,
             },
             opts_size
         )