Merge #1911

1911: set PR_SET_IO_FLUSHER for io-engine  r=daclash a=daclash

This pull request introduces a new functionality to set the PR_SET_IO_FLUSHER
flag in io-engine for improved I/O process handling.  The changes add a new
module that implements a helper function to invoke the prctl system call with
the PR_SET_IO_FLUSHER flag and integrates this call into the io-engine startup
routine. This ensures that processes involved in I/O operations are configured
correctly.

Follow-on changes:

- Update runner scripts to include CAP_SYS_RESOURCE capability
- Update systemd service examples in documentation
- Required for PR_SET_IO_FLUSHER system call in direct execution contexts


Co-authored-by: Dan Clash <[email protected]>

Author: mayastor-bors

doc/run.md

@@ -230,7 +230,7 @@ systemd.services.mayastor = {
     Type = "forking";
     User = "nobody";
     ExecStart = "${pkgs.mayastor}/bin/mayastor";
-    AmbientCapabilities = "CAP_SETPCAP CAP_SYS_ADMIN CAP_IPC_LOCK CAP_SYS_NICE";
+    AmbientCapabilities = "CAP_SETPCAP CAP_SYS_ADMIN CAP_IPC_LOCK CAP_SYS_NICE CAP_SYS_RESOURCE";
   };
 };
 ```
@@ -245,7 +245,7 @@ After=network.target
 Description=A cloud native declarative data plane.
 
 [Service]
-AmbientCapabilities=CAP_SETPCAP CAP_SYS_ADMIN CAP_IPC_LOCK CAP_SYS_NICE
+AmbientCapabilities=CAP_SETPCAP CAP_SYS_ADMIN CAP_IPC_LOCK CAP_SYS_NICE CAP_SYS_RESOURCE
 ExecStart=/usr/bin/mayastor
 Type=simple
 User=nobody

io-engine-bench/.cargo/runner.sh

@@ -20,10 +20,10 @@ fi
 #
 # * Set `cap_setpcap` to be able to set [ambient capabilities](https://lwn.net/Articles/636533/) which can be inherited
 # by children.
-# * Set `cap_sys_admin,cap_ipc_lock,cap_sys_nice` as they are required by the io-engine.
+# * Set `cap_sys_admin,cap_ipc_lock,cap_sys_nice,cap_sys_resource` as they are required by the io-engine.
 ${MAYBE_SUDO} capsh \
-  --caps="cap_setpcap+iep cap_sys_admin,cap_ipc_lock,cap_sys_nice+iep" \
-  --addamb=cap_sys_admin --addamb=cap_ipc_lock --addamb=cap_sys_nice \
+  --caps="cap_setpcap+iep cap_sys_admin,cap_ipc_lock,cap_sys_nice,cap_sys_resource+iep" \
+  --addamb=cap_sys_admin --addamb=cap_ipc_lock --addamb=cap_sys_nice --addamb=cap_sys_resource \
   -- -c "${ARGS}"
 
 if [ -d "$TARGET_CRITERION" ]; then

io-engine/.cargo/runner.sh

@@ -13,8 +13,8 @@ fi
 #
 # * Set `cap_setpcap` to be able to set [ambient capabilities](https://lwn.net/Articles/636533/) which can be inherited
 # by children.
-# * Set `cap_sys_admin,cap_ipc_lock,cap_sys_nice` as they are required by `mayastor`.
+# * Set `cap_sys_admin,cap_ipc_lock,cap_sys_nice,cap_sys_resource` as they are required by `mayastor`.
 ${MAYBE_SUDO} capsh \
-  --caps="cap_setpcap+iep cap_sys_admin,cap_ipc_lock,cap_sys_nice+iep" \
-  --addamb=cap_sys_admin --addamb=cap_ipc_lock --addamb=cap_sys_nice \
+  --caps="cap_setpcap+iep cap_sys_admin,cap_ipc_lock,cap_sys_nice,cap_sys_resource+iep" \
+  --addamb=cap_sys_admin --addamb=cap_ipc_lock --addamb=cap_sys_nice --addamb=cap_sys_resource \
   -- -c "${ARGS}"

io-engine/src/bin/io-engine.rs

@@ -21,6 +21,7 @@ use io_engine::{
     eventing::Event,
     grpc, logger,
     persistent_store::PersistentStoreBuilder,
+    prctl::Prctl,
     subsys::Registration,
 };
 use version_info::fmt_package_info;
@@ -273,6 +274,12 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     info!("{}", fmt_package_info!());
 
+    if let Err(err) = Prctl::set_io_flusher() {
+        error!("Failed to set PR_SET_IO_FLUSHER, CAP_SYS_RESOURCE is required error: {err}");
+    } else {
+        info!("PR_SET_IO_FLUSHER is configured");
+    }
+
     // Handle diagnostics-related commands before initializing the agent.
     // Once diagnostics command is executed (regardless of status), exit the
     // agent.

io-engine/src/lib.rs

@@ -30,6 +30,7 @@ pub mod lvm;
 pub mod lvs;
 pub mod persistent_store;
 pub mod pool_backend;
+pub mod prctl;
 pub mod rebuild;
 pub mod replica_backend;
 pub mod sleep;

io-engine/src/prctl.rs

@@ -0,0 +1,22 @@
+use libc::c_int;
+use std::io::{Error, Result};
+
+const PR_SET_IO_FLUSHER: c_int = 57;
+
+pub struct Prctl;
+
+impl Prctl {
+    /// From man page of prctl: If a user process is involved in the block layer
+    /// or filesystem I/O path, and can allocate memory while processing I/O requests
+    /// it must set arg2 to 1. This will put the process in the IO_FLUSHER  state,
+    /// which allows it special treatment to make progress when allocating memory.
+    /// If arg2 is 0, the process will clear the IO_FLUSHER state, and the default
+    /// behavior will be used.
+    pub fn set_io_flusher() -> Result<()> {
+        let ret = unsafe { libc::prctl(PR_SET_IO_FLUSHER, 1, 0, 0, 0) };
+        if ret != 0 {
+            return Err(Error::last_os_error());
+        }
+        Ok(())
+    }
+}