Use atomic-maybe-uninit's atomic memcpy API

Author: taiki-e

crossbeam-deque/Cargo.toml

@@ -27,6 +27,7 @@ std = ["crossbeam-epoch/std", "crossbeam-utils/std"]
 [dependencies]
 crossbeam-epoch = { version = "0.9.17", path = "../crossbeam-epoch", default-features = false }
 crossbeam-utils = { version = "0.8.18", path = "../crossbeam-utils", default-features = false }
+atomic-maybe-uninit = { version = "0.3.15", git = "https://github.com/taiki-e/atomic-maybe-uninit", branch = "memcpy" }
 
 [dev-dependencies]
 fastrand = "2"

crossbeam-deque/build.rs

@@ -0,0 +1,15 @@
+// The rustc-cfg emitted by the build script are *not* public API.
+
+use std::env;
+
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rustc-check-cfg=cfg(crossbeam_sanitize_any)");
+
+    // `cfg(sanitize = "..")` is not stabilized.
+    if let Ok(sanitize) = env::var("CARGO_CFG_SANITIZE") {
+        if !sanitize.is_empty() {
+            println!("cargo:rustc-cfg=crossbeam_sanitize_any");
+        }
+    }
+}

crossbeam-deque/src/deque.rs

@@ -9,6 +9,7 @@ use std::ptr;
 use std::sync::atomic::{self, AtomicIsize, AtomicPtr, AtomicUsize, Ordering};
 use std::sync::Arc;
 
+use atomic_maybe_uninit::PerByteAtomicMaybeUninit;
 use crossbeam_epoch::{self as epoch, Atomic, Owned};
 use crossbeam_utils::{Backoff, CachePadded};
 
@@ -41,7 +42,7 @@ impl<T> Buffer<T> {
 
         let ptr = Box::into_raw(
             (0..cap)
-                .map(|_| MaybeUninit::<T>::uninit())
+                .map(|_| PerByteAtomicMaybeUninit::new(MaybeUninit::<T>::uninit()))
                 .collect::<Box<[_]>>(),
         )
         .cast::<T>();
@@ -68,22 +69,37 @@ impl<T> Buffer<T> {
     }
 
     /// Writes `task` into the specified `index`.
-    ///
-    /// This method might be concurrently called with another `read` at the same index, which is
-    /// technically speaking a data race and therefore UB. We should use an atomic store here, but
-    /// that would be more expensive and difficult to implement generically for all types `T`.
-    /// Hence, as a hack, we use a volatile write instead.
     unsafe fn write(&self, index: isize, task: MaybeUninit<T>) {
+        atomic_maybe_uninit::cfg_has_atomic_memcpy! {
+            if cfg!(not(any(miri, crossbeam_sanitize_any))) {
+                unsafe {
+                    (*self.at(index).cast::<PerByteAtomicMaybeUninit<T>>()).store(task, Ordering::Relaxed);
+                }
+                return;
+            }
+        }
+        // This method might be concurrently called with another `read` at the same index, which is
+        // technically speaking a data race and therefore UB. We should use an atomic store here
+        // like the code above, but inline assembly is not stable on some tier 2/3 targets and
+        // unsupported in Miri/Sanitizer. Hence, as a hack, we use a volatile write instead.
+        // See also https://github.com/rust-lang/rfcs/pull/3301.
         unsafe { ptr::write_volatile(self.at(index).cast::<MaybeUninit<T>>(), task) }
     }
 
     /// Reads a task from the specified `index`.
-    ///
-    /// This method might be concurrently called with another `write` at the same index, which is
-    /// technically speaking a data race and therefore UB. We should use an atomic load here, but
-    /// that would be more expensive and difficult to implement generically for all types `T`.
-    /// Hence, as a hack, we use a volatile load instead.
     unsafe fn read(&self, index: isize) -> MaybeUninit<T> {
+        atomic_maybe_uninit::cfg_has_atomic_memcpy! {
+            if cfg!(not(any(miri, crossbeam_sanitize_any))) {
+                unsafe {
+                    return (*self.at(index).cast::<PerByteAtomicMaybeUninit<T>>()).load(Ordering::Relaxed);
+                }
+            }
+        }
+        // This method might be concurrently called with another `write` at the same index, which is
+        // technically speaking a data race and therefore UB. We should use an atomic load here
+        // like the code above, but inline assembly is not stable on some tier 2/3 targets and
+        // unsupported in Miri/Sanitizer. Hence, as a hack, we use a volatile write instead.
+        // See also https://github.com/rust-lang/rfcs/pull/3301.
         unsafe { ptr::read_volatile(self.at(index).cast::<MaybeUninit<T>>()) }
     }
 }

crossbeam-utils/Cargo.toml

@@ -30,7 +30,7 @@ std = []
 atomic = ["atomic-maybe-uninit"]
 
 [dependencies]
-atomic-maybe-uninit = { version = "0.3.4", optional = true }
+atomic-maybe-uninit = { version = "0.3.15", optional = true, git = "https://github.com/taiki-e/atomic-maybe-uninit", branch = "memcpy" }
 
 # Enable the use of loom for concurrency testing.
 #

crossbeam-utils/build.rs

@@ -17,7 +17,7 @@ include!("build-common.rs");
 
 fn main() {
     println!("cargo:rerun-if-changed=no_atomic.rs");
-    println!("cargo:rustc-check-cfg=cfg(crossbeam_no_atomic,crossbeam_sanitize_thread,crossbeam_atomic_cell_force_fallback)");
+    println!("cargo:rustc-check-cfg=cfg(crossbeam_no_atomic,crossbeam_sanitize_thread,crossbeam_sanitize_any,crossbeam_atomic_cell_force_fallback)");
 
     let target = match env::var("TARGET") {
         Ok(target) => convert_custom_linux_target(target),
@@ -43,6 +43,9 @@ fn main() {
         if sanitize.contains("thread") {
             println!("cargo:rustc-cfg=crossbeam_sanitize_thread");
         }
-        println!("cargo:rustc-cfg=crossbeam_atomic_cell_force_fallback");
+        if !sanitize.is_empty() {
+            println!("cargo:rustc-cfg=crossbeam_sanitize_any");
+            println!("cargo:rustc-cfg=crossbeam_atomic_cell_force_fallback");
+        }
     }
 }

crossbeam-utils/src/atomic/atomic_cell.rs

@@ -1121,13 +1121,28 @@ where
 
             // Try doing an optimistic read first.
             if let Some(stamp) = lock.optimistic_read() {
-                // We need a volatile read here because other threads might concurrently modify the
-                // value. In theory, data races are *always* UB, even if we use volatile reads and
-                // discard the data when a data race is detected. The proper solution would be to
-                // do atomic reads and atomic writes, but we can't atomically read and write all
-                // kinds of data since `AtomicU8` is not available on stable Rust yet.
-                // Load as `MaybeUninit` because we may load a value that is not valid as `T`.
-                let val = unsafe { ptr::read_volatile(src.cast::<MaybeUninit<T>>()) };
+                atomic_maybe_uninit::cfg_has_atomic_memcpy! {
+                    use atomic_maybe_uninit::PerByteAtomicMaybeUninit;
+                    let val = if cfg!(not(any(miri, crossbeam_sanitize_any))) {
+                        unsafe {
+                            (*src.cast::<PerByteAtomicMaybeUninit<T>>()).load(Ordering::Relaxed)
+                        }
+                    } else {
+                        // See the comment in cfg_no_atomic_memcpy case.
+                        unsafe { ptr::read_volatile(src.cast::<MaybeUninit<T>>()) }
+                    };
+                };
+                atomic_maybe_uninit::cfg_no_atomic_memcpy! {
+                    // We need a volatile read here because other threads might concurrently modify the
+                    // value. In theory, data races are *always* UB, even if we use volatile reads and
+                    // discard the data when a data race is detected. The proper solution would be to
+                    // do atomic reads and atomic writes like the code above, but inline assembly is
+                    // not stable on some tier 2/3 targets and unsupported in Miri/Sanitizer. Hence,
+                    // as a hack, we use a volatile write instead.
+                    // See also https://github.com/rust-lang/rfcs/pull/3301.
+                    // Load as `MaybeUninit` because we may load a value that is not valid as `T`.
+                    let val = unsafe { ptr::read_volatile(src.cast::<MaybeUninit<T>>()) };
+                }
 
                 if lock.validate_read(stamp) {
                     return unsafe { val.assume_init() };