Merge #1902

1902: ci: update github runners to oci gh arc runners r=tiagolobocastro a=tiagolobocastro

CNCF has hosted ephemeral GitHub runners on Oracle instead of the GitHub-hosted ones, which now incur a cost to use.

Co-authored-by: mayastor-bors <[email protected]>

Author: mayastor-bors

.github/workflows/bdd.yml

@@ -9,7 +9,7 @@ env:
 
 jobs:
   bdd-tests:
-    runs-on: ubuntu-latest-16-cores
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     permissions:
       contents: read
       id-token: write
@@ -38,7 +38,7 @@ jobs:
 
       - name: Pre-populate nix-shell
         run: |
-          export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r)
+          export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' spdk-rs/nix/sources.json -r)
           echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV
           nix-shell --run "echo" shell.nix
 
@@ -61,6 +61,8 @@ jobs:
           done
           # for the coredump check
           sudo apt-get install gdb
+          # Check if nvme config is valid, otherwise apply it
+          sudo ./scripts/nvme-conf.sh --check --apply --overwrite
 
       - name: Setup VENV
         run: nix-shell --run "./test/python/setup.sh"

.github/workflows/unit-int.yml

@@ -9,7 +9,7 @@ env:
 
 jobs:
   int-tests:
-    runs-on: ubuntu-latest-16-cores
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     permissions:
       contents: read
       id-token: write
@@ -38,7 +38,7 @@ jobs:
 
       - name: Pre-populate nix-shell
         run: |
-          export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r)
+          export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' spdk-rs/nix/sources.json -r)
           echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV
           nix-shell --run "echo" shell.nix
 
@@ -62,6 +62,8 @@ jobs:
             sudo modprobe $module
           done
           sudo apt-get install gdb
+          # Check if nvme config is valid, otherwise apply it
+          sudo ./scripts/nvme-conf.sh --check --apply --overwrite
 
       - name: Run Rust Tests
         run: |

Cargo.lock

@@ -38,9 +38,29 @@ RUST_LOG=TRACE cargo test --features=io-engine-testing -- --test-threads 1 --noc
 > _**NOTE**_:
 > The flag --features=io-engine-testing ensures you run tests with features enabled only for testing purposes
 
+Recent linux kernel versions no longer allow for mixing of nvme hostid's with different nvme hostnqn's, so you should
+ensure your system is configured with default values.
+Typically, this happens as long as you install nvme-cli on your host, but you can also ensure this by using our script:
+
+```bash
+sudo ./scripts/nvme-conf.sh --check --apply
+ NVME SYSCONFDIR   : /etc/nvme
+|           exists : true
+|= hostid          : /etc/nvme/hostid
+|           exists : true
+|          content : 7b562a4c-61e8-ef10-a994-047c16834e32
+|= hostnqn         : /etc/nvme/hostnqn
+|           exists : true
+|          content : nqn.2014-08.org.nvmexpress:uuid:7b562a4c-61e8-ef10-a994-047c16834e32
+
+
+INFO: /etc/nvme is valid
+```
+
 ## Testing your own SPDK version
 
-To test your custom SPDK version please refer to the [spdk-rs documentation](https://github.com/openebs/spdk-rs/blob/develop/README.md#custom-spdk)
+To test your custom SPDK version please refer to
+the [spdk-rs documentation](https://github.com/openebs/spdk-rs/blob/develop/README.md#custom-spdk)
 
 ## Using PCIe NVMe devices in cargo tests while developing
 

doc/test.md

@@ -88,8 +88,27 @@ pub fn nvme_connect(
     transport: &str,
     must_succeed: bool,
 ) -> ExitStatus {
-    let status = Command::new("nvme")
-        .args(["connect"])
+    let mut comand = Command::new("nvme");
+    let mut comand = comand.args(["connect"]);
+
+    if !std::path::Path::new("/etc/nvme/hostid").exists()
+        || !std::path::Path::new("/etc/nvme/hostnqn").exists()
+    {
+        match (
+            std::env::var("NVME_HOSTID").ok(),
+            std::env::var("NVME_HOSTNQN").ok(),
+        ) {
+            (Some(hid), Some(hnqn)) => {
+                tracing::warn!("/etc/nvme is not present, using {hid} and {hnqn}");
+                comand = comand.args(["-I", &hid]).args(["-q", &hnqn])
+            }
+            _ => {
+                panic!("/etc/nvme is not present and no env var NVME_HOSTID available")
+            }
+        }
+    }
+
+    let status = comand
         .args(["-t", transport])
         .args(["-a", target_addr])
         .args(["-s", "8420"])

io-engine-tests/src/nvme.rs

@@ -211,6 +211,7 @@ impl<'n> Nexus<'n> {
             None => self.uuid().to_string(),
         };
 
+        // these are actually tries, not retries!
         let mut retry = PersistentStore::retries();
         let mut logged = false;
         loop {

io-engine/src/bdev/nexus/nexus_persistence.rs

@@ -273,7 +273,7 @@ async fn nexus_child_retire_persist_unresponsive_with_fio() {
     cluster.test.thaw("etcd").await.unwrap();
 
     assert!(
-        tokio::time::timeout(Duration::from_secs(1), &mut r1)
+        tokio::time::timeout(Duration::from_secs(2), &mut r1)
             .await
             .is_ok(),
         "I/O to nexus must proceed when ETCD is thawed"
@@ -334,15 +334,16 @@ const NEXUS_NAME: &str = "nexus_0";
 const NEXUS_UUID: &str = "cdc2a7db-3ac3-403a-af80-7fadc1581c47";
 
 #[tokio::test]
+#[ignore]
 /// Test ETCD misbehaviour during a child retirement: a nexus must not ack I/Os
 /// to a client if a persistent store cannot be updated while a child is being
 /// retired.
 ///
 /// [1] Create etcd, pools, replicas, and nexus.
-/// [2] Inject an fault to a replica.
+/// [2] Inject a fault to a replica.
 /// [3] Pause ETCD container.
 /// [4] Write to the nexus. A replica fails due to injected fault, and I/O on
-///     nexus must stuck.
+///     nexus must remain stuck.
 /// [5] Thaw ETCD container.
 /// [6] I/Os must now be acknowledged to the client.
 async fn nexus_child_retire_persist_unresponsive_with_bdev_io() {
@@ -399,7 +400,7 @@ async fn nexus_child_retire_persist_unresponsive_with_bdev_io() {
         "I/O to nexus must proceed when ETCD is thawed"
     );
 
-    // Check that 1st child is fauled, and 2nd is open.
+    // Check that 1st child is faulted, and 2nd is open.
     assert!(matches!(
         nex.child_at(0).state(),
         ChildState::Faulted(FaultReason::IoError)

io-engine/tests/nexus_child_retire.rs

@@ -28,3 +28,7 @@ version = "1.0"
 [dependencies.udev]
 features = ["hwdb", "mio"]
 version = "^0.9.1"
+
+[dependencies.uuid]
+features = ["v4"]
+version = "1.11.0"

libnvme-rs/Cargo.toml

@@ -1,5 +1,4 @@
 use std::{convert::TryFrom, io, os::raw::c_char, time::Duration};
-
 use url::{ParseError, Url};
 
 use mio::{Events, Interest, Poll, Token};
@@ -127,15 +126,28 @@ impl NvmeTarget {
     /// Returns Ok on successful connect
     pub fn connect(&self) -> Result<(), NvmeError> {
         let r = NvmeRoot::new(unsafe { crate::nvme_scan(std::ptr::null()) });
+
+        // Note, if host id and hostnqn are not present in the system, then hostid is generated at
+        // the kernel but becomes sticky and there seems no way of querying it.
         let hostid = NvmeStringWrapper::new(unsafe { crate::nvmf_hostid_from_file() });
+        let hostid_gen = std::ffi::CString::new(uuid::Uuid::new_v4().to_string()).unwrap();
 
         let hostnqn = match self.hostnqn_autogen {
             true => NvmeStringWrapper::new(unsafe { crate::nvmf_hostnqn_generate() }),
             false => NvmeStringWrapper::new(unsafe { crate::nvmf_hostnqn_from_file() }),
         };
 
-        let h =
-            unsafe { crate::nvme_lookup_host(r.as_mut_ptr(), hostnqn.as_ptr(), hostid.as_ptr()) };
+        let h = unsafe {
+            crate::nvme_lookup_host(
+                r.as_mut_ptr(),
+                hostnqn.as_ptr(),
+                if self.hostnqn_autogen {
+                    hostid_gen.as_ptr()
+                } else {
+                    hostid.as_ptr()
+                },
+            )
+        };
         if h.is_null() {
             return Err(NvmeError::LookupHostError { rc: -libc::ENOMEM });
         }

libnvme-rs/src/nvme_uri.rs

@@ -17,7 +17,7 @@ cleanup_handler
 trap cleanup_handler INT QUIT TERM HUP EXIT
 
 export PATH=$PATH:${HOME}/.cargo/bin
-set -euxo pipefail
+set -euo pipefail
 
 # Warn if rdma-rxe and nvme-rdme kernel modules are not
 # available. Absence of rdma-rxe can be ignored on hardware
@@ -30,6 +30,11 @@ if ! lsmod | grep -q nvme_rdma; then
   echo "Warning: nvme_rdma kernel module is not loaded. Please load it for rdma tests to work."
 fi
 
+if ! "$SCRIPTDIR"/nvme-conf.sh --check; then
+  echo "Warning: nvme configuration may not be valid, this can cause some test issues"
+  exit 1
+fi
+
 ( cd jsonrpc && cargo test )
 # test dependencies
 cargo build --bins --features=io-engine-testing