refactor(driver): get rid of pin

Author: George-Miao

compio-driver/Cargo.toml

@@ -23,11 +23,11 @@ cfg-if = { workspace = true }
 flume = { workspace = true, default-features = false }
 futures-util = { workspace = true }
 socket2 = { workspace = true, features = ["all"] }
-pin-project-lite = { workspace = true }
 thin-cell = { workspace = true }
 smallvec = { workspace = true, optional = true, features = ["union"] }
 synchrony = { workspace = true, features = ["waker_slot"] }
 bitflags = { version = "2.11.0" }
+paste = { workspace = true }
 
 # Windows specific dependencies
 [target.'cfg(windows)'.dependencies]
@@ -52,7 +52,6 @@ io-uring = { version = "0.7.0", optional = true }
 io_uring_buf_ring = { version = "0.2.2", optional = true }
 once_cell = { workspace = true, optional = true }
 polling = { version = "3.3.0", optional = true }
-paste = { workspace = true }
 slab = { workspace = true, optional = true }
 
 [dev-dependencies]

compio-driver/build.rs

@@ -14,6 +14,7 @@ fn main() {
         freebsd: { target_os = "freebsd" },
         solarish: { any(target_os = "illumos", target_os = "solaris") },
         aio: { any(freebsd, solarish) },
+        polling: { all(unix, feature = "polling") },
         io_uring: { all(target_os = "linux", feature = "io-uring") },
         fusion: { all(target_os = "linux", feature = "io-uring", feature = "polling") },
 

compio-driver/src/control.rs

@@ -0,0 +1,136 @@
+use std::mem::MaybeUninit;
+
+use compio_buf::IntoInner;
+
+use crate::{DriverType, OpCode};
+
+cfg_if::cfg_if! {
+    if #[cfg(fusion)] {
+        use crate::{PollOpCode, IourOpCode};
+    } else if #[cfg(io_uring)] {
+        use crate::OpCode as IourOpCode;
+    } else if #[cfg(polling)]{
+        use crate::OpCode as PollOpCode;
+    }
+}
+
+#[cfg(not(fusion))]
+type ControlInner<T> = <T as OpCode>::Control;
+
+#[cfg(fusion)]
+enum ControlInner<T: OpCode + ?Sized> {
+    Poll(<T as PollOpCode>::Control),
+    IoUring(<T as IourOpCode>::Control),
+}
+
+#[cfg(fusion)]
+impl<T: OpCode> ControlInner<T> {
+    pub fn iour(&mut self) -> &mut <T as IourOpCode>::Control {
+        match self {
+            ControlInner::Poll(_) => unreachable!("Current driver is not `io-uring`"),
+            ControlInner::IoUring(control) => control,
+        }
+    }
+
+    pub fn poll(&mut self) -> &mut <T as PollOpCode>::Control {
+        match self {
+            ControlInner::Poll(control) => control,
+            ControlInner::IoUring(_) => unreachable!("Current driver is not `polling`"),
+        }
+    }
+}
+
+/// A utility type that put a [`OpCode`] and its [`OpCode::Control`] together.
+///
+/// The only way to access this type is through [`ThinCell`], which pins it on
+/// the heap and guarantees any self-referential pointers to be valid.
+pub(crate) struct Carrier<T: OpCode + ?Sized> {
+    control: MaybeUninit<ControlInner<T>>,
+    op: T,
+}
+
+impl<T: OpCode> IntoInner for Carrier<T> {
+    type Inner = T;
+
+    fn into_inner(self) -> Self::Inner {
+        self.op
+    }
+}
+
+impl<T: OpCode> Carrier<T> {
+    pub fn new(op: T) -> Self {
+        Self {
+            control: MaybeUninit::uninit(),
+            op,
+        }
+    }
+
+    /// Init the Carrier
+    ///
+    /// # Safety
+    ///
+    /// self must have stable address until control is no longer needed, which
+    /// include all function calls via `dyn Carry`.
+    pub unsafe fn init(&mut self, driver_ty: DriverType) {
+        #[cfg(fusion)]
+        {
+            let control = match driver_ty {
+                DriverType::Poll => ControlInner::Poll(unsafe { PollOpCode::init(&mut self.op) }),
+                DriverType::IoUring => {
+                    ControlInner::IoUring(unsafe { IourOpCode::init(&mut self.op) })
+                }
+                DriverType::IOCP => unreachable!("Cannot be windows"),
+            };
+
+            self.control.write(control);
+        }
+
+        #[cfg(not(fusion))]
+        {
+            _ = driver_ty;
+
+            let control = unsafe { OpCode::init(&mut self.op) };
+            self.control.write(control);
+        }
+    }
+
+    #[cfg(io_uring)]
+    pub fn as_iour(&mut self) -> (&mut T, &mut <T as IourOpCode>::Control) {
+        let control = unsafe { self.control.assume_init_mut() };
+        #[cfg(fusion)]
+        {
+            (&mut self.op, control.iour())
+        }
+        #[cfg(not(fusion))]
+        {
+            (&mut self.op, control)
+        }
+    }
+
+    #[cfg(polling)]
+    pub fn as_poll(&mut self) -> (&mut T, &mut <T as PollOpCode>::Control) {
+        let control = unsafe { self.control.assume_init_mut() };
+        #[cfg(fusion)]
+        {
+            (&mut self.op, control.poll())
+        }
+        #[cfg(not(fusion))]
+        {
+            (&mut self.op, control)
+        }
+    }
+
+    #[cfg(windows)]
+    pub fn as_iocp(&self) -> (&T, &<T as OpCode>::Control) {
+        let control = unsafe { self.control.assume_init_ref() };
+
+        (&self.op, control)
+    }
+
+    #[cfg(windows)]
+    pub fn as_iocp_mut(&mut self) -> (&mut T, &mut <T as OpCode>::Control) {
+        let control = unsafe { self.control.assume_init_mut() };
+
+        (&mut self.op, control)
+    }
+}

compio-driver/src/key.rs

@@ -6,21 +6,20 @@ use std::{
     io,
     mem::{self, ManuallyDrop},
     ops::{Deref, DerefMut},
-    pin::Pin,
     task::Waker,
 };
 
-use compio_buf::BufResult;
-use thin_cell::unsync::{Ref, ThinCell};
+use compio_buf::{BufResult, IntoInner};
+use thin_cell::unsync::{Inner, Ref, ThinCell};
 
-use crate::{Extra, OpCode, PushEntry};
+use crate::{Carry, DriverType, Extra, OpCode, PushEntry, control::Carrier};
 
 /// An operation with other needed information.
 ///
 /// You should not use `RawOp` directly. Instead, use [`Key`] to manage the
 /// reference-counted pointer to it.
 #[repr(C)]
-pub(crate) struct RawOp<T: ?Sized> {
+pub(crate) struct RawOp<M: ?Sized> {
     // Platform-specific extra data.
     //
     // - On Windows, it holds the `OVERLAPPED` buffer and a pointer to the driver.
@@ -33,10 +32,10 @@ pub(crate) struct RawOp<T: ?Sized> {
     // The cancelled flag indicates the op has been cancelled.
     cancelled: bool,
     result: PushEntry<Option<Waker>, io::Result<usize>>,
-    pub(crate) op: T,
+    pub(crate) Carrier: M,
 }
 
-impl<T: ?Sized> RawOp<T> {
+impl<M: ?Sized> RawOp<M> {
     pub fn extra(&self) -> &Extra {
         &self.extra
     }
@@ -45,11 +44,6 @@ impl<T: ?Sized> RawOp<T> {
         &mut self.extra
     }
 
-    fn pinned_op(&mut self) -> Pin<&mut T> {
-        // SAFETY: inner is always pinned with ThinCell.
-        unsafe { Pin::new_unchecked(&mut self.op) }
-    }
-
     #[cfg(io_uring)]
     pub fn wake_by_ref(&mut self) {
         if let PushEntry::Pending(Some(w)) = &self.result {
@@ -59,7 +53,7 @@ impl<T: ?Sized> RawOp<T> {
 }
 
 #[cfg(windows)]
-impl<T: OpCode + ?Sized> RawOp<T> {
+impl<M: crate::Carry + ?Sized> RawOp<M> {
     /// Call [`OpCode::operate`] and assume that it is not an overlapped op,
     /// which means it never returns [`Poll::Pending`].
     ///
@@ -68,22 +62,21 @@ impl<T: OpCode + ?Sized> RawOp<T> {
         use std::task::Poll;
 
         let optr = self.extra_mut().optr();
-        let op = self.pinned_op();
-        let res = unsafe { op.operate(optr.cast()) };
+        let res = unsafe { self.Carrier.operate(optr.cast()) };
         match res {
             Poll::Pending => unreachable!("this operation is not overlapped"),
             Poll::Ready(res) => res,
         }
     }
 }
 
-impl<T: ?Sized> Debug for RawOp<T> {
+impl<M: ?Sized> Debug for RawOp<M> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("RawOp")
             .field("extra", &self.extra)
             .field("cancelled", &self.cancelled)
             .field("result", &self.result)
-            .field("op", &"<...>")
+            .field("Carrier", &"<...>")
             .finish()
     }
 }
@@ -120,7 +113,9 @@ impl<T> Key<T> {
     pub(crate) fn erase(self) -> ErasedKey {
         self.erased
     }
+}
 
+impl<T: OpCode> Key<T> {
     /// Take the inner result if it is completed.
     ///
     /// # Panics
@@ -135,8 +130,8 @@ impl<T> Key<T> {
 
 impl<T: OpCode + 'static> Key<T> {
     /// Create [`RawOp`] and get the [`Key`] to it.
-    pub(crate) fn new(op: T, extra: impl Into<Extra>) -> Self {
-        let erased = ErasedKey::new(op, extra.into());
+    pub(crate) fn new(op: T, extra: impl Into<Extra>, driver_ty: DriverType) -> Self {
+        let erased = ErasedKey::new(op, extra.into(), driver_ty);
 
         Self {
             erased,
@@ -171,7 +166,7 @@ impl<T> DerefMut for Key<T> {
 #[derive(Clone)]
 #[repr(transparent)]
 pub struct ErasedKey {
-    inner: ThinCell<RawOp<dyn OpCode>>,
+    inner: ThinCell<RawOp<dyn Carry>>,
 }
 
 impl PartialEq for ErasedKey {
@@ -199,16 +194,22 @@ impl std::borrow::Borrow<usize> for ErasedKey {
 
 impl ErasedKey {
     /// Create [`RawOp`] and get the [`ErasedKey`] to it.
-    pub(crate) fn new<T: OpCode + 'static>(op: T, extra: Extra) -> Self {
+    pub(crate) fn new<T: OpCode + 'static>(op: T, extra: Extra, driver_ty: DriverType) -> Self {
         let raw_op = RawOp {
             extra,
             cancelled: false,
             result: PushEntry::Pending(None),
-            op,
+            Carrier: Carrier::new(op),
         };
-        // SAFETY: Unsize coersion from `RawOp<T>` to `RawOp<dyn OpCode>`
-        let inner = unsafe { ThinCell::new_unsize(raw_op, |p| p as _) };
-        Self { inner }
+        let mut inner = ThinCell::new(raw_op);
+        // SAFETY:
+        // - ThinCell is just created, there will be no shared owner or borrower
+        // - Carrier is being pinned by ThinCell, it will have a stable address until
+        //   move out
+        unsafe { inner.borrow_unchecked().Carrier.init(driver_ty) };
+        Self {
+            inner: unsafe { inner.unsize(|p| p as *const Inner<RawOp<dyn Carry>>) },
+        }
     }
 
     /// Create from `user_data` pointer.
@@ -255,7 +256,7 @@ impl ErasedKey {
     }
 
     #[inline]
-    pub(crate) fn borrow(&self) -> Ref<'_, RawOp<dyn OpCode>> {
+    pub(crate) fn borrow(&self) -> Ref<'_, RawOp<dyn Carry>> {
         self.inner.borrow()
     }
 
@@ -283,7 +284,7 @@ impl ErasedKey {
             let this = &mut *this;
             if this.extra.is_iour() {
                 unsafe {
-                    Pin::new_unchecked(&mut this.op).set_result(&res, &this.extra);
+                    this.Carrier.set_result(&res, &this.extra);
                 }
             }
         }
@@ -323,12 +324,12 @@ impl ErasedKey {
     ///
     /// Panics if the result is not ready or the `Key` is not unique (multiple
     /// references or borrowed).
-    unsafe fn take_result<T>(self) -> BufResult<usize, T> {
+    unsafe fn take_result<T: OpCode>(self) -> BufResult<usize, T> {
         // SAFETY: Caller guarantees that `T` is the actual concrete type.
-        let this = unsafe { self.inner.downcast_unchecked::<RawOp<T>>() };
+        let this = unsafe { self.inner.downcast_unchecked::<RawOp<Carrier<T>>>() };
         let op = this.try_unwrap().map_err(|_| ()).expect("Key not unique");
         let res = op.result.take_ready().expect("Result not ready");
-        BufResult(res, op.op)
+        BufResult(res, op.Carrier.into_inner())
     }
 
     /// Unsafely freeze the `Key` by bypassing borrow flag of [`ThinCell`],
@@ -363,14 +364,10 @@ pub(crate) struct FrozenKey {
 }
 
 impl FrozenKey {
-    pub fn as_mut(&mut self) -> &mut RawOp<dyn OpCode> {
+    pub fn as_mut(&mut self) -> &mut RawOp<dyn Carry> {
         unsafe { self.inner.inner.borrow_unchecked() }
     }
 
-    pub fn pinned_op(&mut self) -> Pin<&mut dyn OpCode> {
-        self.as_mut().pinned_op()
-    }
-
     pub fn into_inner(self) -> ErasedKey {
         ManuallyDrop::into_inner(self.inner)
     }
@@ -405,29 +402,23 @@ impl Deref for BorrowedKey {
     }
 }
 
-pub trait RefExt {
-    fn pinned_op(&mut self) -> Pin<&mut dyn OpCode>;
-}
-
-impl RefExt for Ref<'_, RawOp<dyn OpCode>> {
-    fn pinned_op(&mut self) -> Pin<&mut dyn OpCode> {
-        self.deref_mut().pinned_op()
-    }
-}
-
 #[cfg(test)]
 mod test {
     use std::borrow::Borrow;
 
     use compio_buf::BufResult;
 
-    use crate::{Proactor, key::ErasedKey, op::Asyncify};
+    use crate::{DriverType, Proactor, key::ErasedKey, op::Asyncify};
 
     #[test]
     fn test_key_borrow() {
         let driver = Proactor::new().unwrap();
         let extra = driver.default_extra();
-        let key = ErasedKey::new(Asyncify::new(|| BufResult(Ok(0), [0u8])), extra);
+        let key = ErasedKey::new(
+            Asyncify::new(|| BufResult(Ok(0), [0u8])),
+            extra,
+            DriverType::Poll,
+        );
         assert_eq!(&key.as_raw(), Borrow::<usize>::borrow(&key));
     }
 }