1414mod tests;
1515
1616mod spin_mutex;
17- mod unsafe_list;
1817
1918use fortanix_sgx_abi:: { EV_UNPARK , Tcs , WAIT_INDEFINITE } ;
2019
21- pub use self :: spin_mutex:: { SpinMutex , SpinMutexGuard , try_lock_or_false} ;
22- use self :: unsafe_list:: { UnsafeList , UnsafeListEntry } ;
20+ pub use self :: spin_mutex:: { SpinMutex , SpinMutexGuard } ;
2321use super :: abi:: { thread, usercalls} ;
2422use crate :: num:: NonZero ;
2523use crate :: ops:: { Deref , DerefMut } ;
2624use crate :: panic:: { self , AssertUnwindSafe } ;
25+ use crate :: pin:: Pin ;
26+ use crate :: sys:: sync:: unsafe_list:: { UnsafeList , UnsafeListEntry } ;
2727use crate :: time:: Duration ;
2828
2929/// An queue entry in a `WaitQueue`.
@@ -38,24 +38,41 @@ struct WaitEntry {
3838/// queue and the data are synchronized, since the type itself is not `Sync`.
3939///
4040/// Consumers of this API should use a synchronization primitive for shared
41- /// access, such as `SpinMutex`.
42- # [ derive ( Default ) ]
41+ /// access. `WaitVariable::new` is the only constructor and provides that
42+ /// with `SpinMutex`.
4343pub struct WaitVariable < T > {
4444 queue : WaitQueue ,
4545 lock : T ,
4646}
4747
4848impl < T > WaitVariable < T > {
49- pub const fn new ( var : T ) -> Self {
50- WaitVariable { queue : WaitQueue :: new ( ) , lock : var }
51- }
52-
5349 pub fn lock_var ( & self ) -> & T {
5450 & self . lock
5551 }
5652
57- pub fn lock_var_mut ( & mut self ) -> & mut T {
58- & mut self . lock
53+ pub fn lock_var_mut ( self : Pin < & mut Self > ) -> & mut T {
54+ // SAFETY: `lock` is not structurally pinned: a pinned `WaitVariable`
55+ // makes no promise that `T` is pinned.
56+ unsafe { & mut self . get_unchecked_mut ( ) . lock }
57+ }
58+
59+ fn queue ( self : Pin < & mut Self > ) -> Pin < & mut WaitQueue > {
60+ // SAFETY: `queue` is structurally pinned: a pinned `WaitVariable`
61+ // pins it, and it is never moved out of it.
62+ unsafe { self . map_unchecked_mut ( |this| & mut this. queue ) }
63+ }
64+
65+ /// Creates a mutex-protected `WaitVariable` on the heap, with its queue's
66+ /// list initialized. Initialization makes the list self-referential and
67+ /// happens before pinning: only the `Box` pointer is moved into the
68+ /// `Pin`, the heap allocation itself never moves.
69+ pub fn new ( value : T ) -> Pin < Box < SpinMutex < WaitVariable < T > > > > {
70+ // SAFETY: `init` is called below, before the queue is otherwise used
71+ // or dropped.
72+ let queue = unsafe { WaitQueue :: new ( ) } ;
73+ let result = Box :: new ( SpinMutex :: new ( WaitVariable { queue, lock : value } ) ) ;
74+ result. lock ( ) . queue . inner . init ( ) ;
75+ Box :: into_pin ( result)
5976 }
6077}
6178
@@ -68,7 +85,7 @@ pub enum NotifiedTcs {
6885/// An RAII guard that will notify a set of target threads as well as unlock
6986/// a mutex on drop.
7087pub struct WaitGuard < ' a , T : ' a > {
71- mutex_guard : Option < SpinMutexGuard < ' a , WaitVariable < T > > > ,
88+ mutex_guard : Option < Pin < SpinMutexGuard < ' a , WaitVariable < T > > > > ,
7289 notified_tcs : NotifiedTcs ,
7390}
7491
@@ -79,21 +96,36 @@ pub struct WaitGuard<'a, T: 'a> {
7996/// safe because the waiting thread will not return from that stack frame until
8097/// after it is notified. The notifying thread ensures to clean up any
8198/// references to the list entries before sending the wakeup event.
99+ // The safety requirements of `UnsafeList` are upheld as follows:
100+ //
101+ // * All list operations are performed while holding the lock of the
102+ // `SpinMutex` around the `WaitVariable` containing the list.
103+ // * A waiting thread pushes a stack-allocated entry and does not invalidate
104+ // it while it is in the list: it only accesses the entry through the
105+ // reference `push` returned, reading `wake` under the `WaitEntry`'s own
106+ // `SpinMutex`.
107+ // * `push` -> `pop`: a notifying thread pops the entry and sets `wake` under
108+ // the `WaitEntry`'s `SpinMutex`; when that mutex is released, the thread
109+ // will no longer access the entry (guaranteed by the mutex guard). The
110+ // waiting thread only returns from the stack frame containing the entry
111+ // once it observes `wake == true` under that same mutex, so the entry is
112+ // only deallocated after the notifying thread's last access to it.
113+ // * `push` -> `remove`: on a timeout, `wait_timeout` re-acquires the queue
114+ // lock and checks `wake`: the entry is still in the list if and only if
115+ // `wake` is not set, because notifying threads always `pop` an entry
116+ // before setting its `wake`. Only if the entry is still in the list is it
117+ // removed.
118+ // * Besides as described, no other exclusive references to the entry are
119+ // taken.
82120pub struct WaitQueue {
83121 // We use an inner Mutex here to protect the data in the face of spurious
84122 // wakeups.
85123 inner : UnsafeList < SpinMutex < WaitEntry > > ,
86124}
87125unsafe impl Send for WaitQueue { }
88126
89- impl Default for WaitQueue {
90- fn default ( ) -> Self {
91- Self :: new ( )
92- }
93- }
94-
95127impl < ' a , T > Deref for WaitGuard < ' a , T > {
96- type Target = SpinMutexGuard < ' a , WaitVariable < T > > ;
128+ type Target = Pin < SpinMutexGuard < ' a , WaitVariable < T > > > ;
97129
98130 fn deref ( & self ) -> & Self :: Target {
99131 self . mutex_guard . as_ref ( ) . unwrap ( )
@@ -118,23 +150,42 @@ impl<'a, T> Drop for WaitGuard<'a, T> {
118150}
119151
120152impl WaitQueue {
121- pub const fn new ( ) -> Self {
122- WaitQueue { inner : UnsafeList :: new ( ) }
153+ /// Creates a new queue.
154+ ///
155+ /// # Safety
156+ ///
157+ /// The caller must initialize the queue's list (`UnsafeList::init`)
158+ /// before any other use of the queue, including dropping it.
159+ /// `WaitVariable::new`, the sole constructor of the containing
160+ /// structure, does this.
161+ pub const unsafe fn new ( ) -> Self {
162+ // SAFETY: the caller upholds `UnsafeList::new`'s contract (see this
163+ // function's safety requirements).
164+ WaitQueue { inner : unsafe { UnsafeList :: new ( ) } }
165+ }
166+
167+ fn inner ( self : Pin < & mut Self > ) -> Pin < & mut UnsafeList < SpinMutex < WaitEntry > > > {
168+ // SAFETY: `inner` is structurally pinned: a pinned `WaitQueue` pins
169+ // it, and it is never moved out of it.
170+ unsafe { self . map_unchecked_mut ( |this| & mut this. inner ) }
123171 }
124172
125173 /// Adds the calling thread to the `WaitVariable`'s wait queue, then wait
126174/// until a wakeup event.
127175///
128176/// This function does not return until this thread has been awoken. When `before_wait` panics,
129177/// this function will abort.
130- pub fn wait < T , F : FnOnce ( ) > ( mut guard : SpinMutexGuard < ' _ , WaitVariable < T > > , before_wait : F ) {
178+ pub fn wait < T , F : FnOnce ( ) > (
179+ mut guard : Pin < SpinMutexGuard < ' _ , WaitVariable < T > > > ,
180+ before_wait : F ,
181+ ) {
131182 // very unsafe: check requirements of UnsafeList::push
132183 unsafe {
133184 let mut entry = UnsafeListEntry :: new ( SpinMutex :: new ( WaitEntry {
134185 tcs : thread:: current ( ) ,
135186 wake : false ,
136187 } ) ) ;
137- let entry = guard. queue . inner . push ( & mut entry) ;
188+ let entry = guard. as_mut ( ) . queue ( ) . inner ( ) . push ( & mut entry) ;
138189 drop ( guard) ;
139190 if let Err ( _e) = panic:: catch_unwind ( AssertUnwindSafe ( || before_wait ( ) ) ) {
140191 rtabort ! ( "Panic before wait on wakeup event" )
@@ -155,7 +206,7 @@ impl WaitQueue {
155206/// If not, it will remove the calling thread from the wait queue.
156207/// When `before_wait` panics, this function will abort.
157208pub fn wait_timeout < T , F : FnOnce ( ) > (
158- lock : & SpinMutex < WaitVariable < T > > ,
209+ lock : Pin < & SpinMutex < WaitVariable < T > > > ,
159210 timeout : Duration ,
160211 before_wait : F ,
161212 ) -> bool {
@@ -165,19 +216,19 @@ impl WaitQueue {
165216 tcs : thread:: current ( ) ,
166217 wake : false ,
167218 } ) ) ;
168- let entry_lock = lock. lock ( ) . queue . inner . push ( & mut entry) ;
219+ let entry_lock = lock. lock_pinned ( ) . as_mut ( ) . queue ( ) . inner ( ) . push ( & mut entry) ;
169220 if let Err ( _e) = panic:: catch_unwind ( AssertUnwindSafe ( || before_wait ( ) ) ) {
170221 rtabort ! ( "Panic before wait on wakeup event or timeout" )
171222 }
172223 usercalls:: wait_timeout ( EV_UNPARK , timeout, || entry_lock. lock ( ) . wake ) ;
173224 // acquire the wait queue's lock first to avoid deadlock
174225 // and ensure no other function can simultaneously access the list
175226 // (e.g., `notify_one` or `notify_all`)
176- let mut guard = lock. lock ( ) ;
227+ let mut guard = lock. lock_pinned ( ) ;
177228 let success = entry_lock. lock ( ) . wake ;
178229 if !success {
179230 // nobody is waking us up, so remove our entry from the wait queue.
180- guard. queue . inner . remove ( & mut entry) ;
231+ guard. as_mut ( ) . queue ( ) . inner ( ) . remove ( & mut entry) ;
181232 }
182233 success
183234 }
@@ -189,14 +240,14 @@ impl WaitQueue {
189240/// If a waiter is found, a `WaitGuard` is returned which will notify the
190241/// waiter when it is dropped.
191242pub fn notify_one < T > (
192- mut guard : SpinMutexGuard < ' _ , WaitVariable < T > > ,
193- ) -> Result < WaitGuard < ' _ , T > , SpinMutexGuard < ' _ , WaitVariable < T > > > {
243+ mut guard : Pin < SpinMutexGuard < ' _ , WaitVariable < T > > > ,
244+ ) -> Result < WaitGuard < ' _ , T > , Pin < SpinMutexGuard < ' _ , WaitVariable < T > > > > {
194245 // SAFETY: lifetime of the pop() return value is limited to the map
195246 // closure (The closure return value is 'static). The underlying
196247 // stack frame won't be freed until after the lock on the queue is released
197248 // (i.e., `guard` is dropped).
198249 unsafe {
199- let tcs = guard. queue . inner . pop ( ) . map ( |entry| -> Tcs {
250+ let tcs = guard. as_mut ( ) . queue ( ) . inner ( ) . pop ( ) . map ( |entry| -> Tcs {
200251 let mut entry_guard = entry. lock ( ) ;
201252 entry_guard. wake = true ;
202253 entry_guard. tcs
@@ -216,14 +267,14 @@ impl WaitQueue {
216267/// If at least one waiter is found, a `WaitGuard` is returned which will
217268/// notify all waiters when it is dropped.
218269pub fn notify_all < T > (
219- mut guard : SpinMutexGuard < ' _ , WaitVariable < T > > ,
220- ) -> Result < WaitGuard < ' _ , T > , SpinMutexGuard < ' _ , WaitVariable < T > > > {
270+ mut guard : Pin < SpinMutexGuard < ' _ , WaitVariable < T > > > ,
271+ ) -> Result < WaitGuard < ' _ , T > , Pin < SpinMutexGuard < ' _ , WaitVariable < T > > > > {
221272 // SAFETY: lifetime of the pop() return values are limited to the
222273 // while loop body. The underlying stack frames won't be freed until
223274 // after the lock on the queue is released (i.e., `guard` is dropped).
224275 unsafe {
225276 let mut count = 0 ;
226- while let Some ( entry) = guard. queue . inner . pop ( ) {
277+ while let Some ( entry) = guard. as_mut ( ) . queue ( ) . inner ( ) . pop ( ) {
227278 count += 1 ;
228279 let mut entry_guard = entry. lock ( ) ;
229280 entry_guard. wake = true ;
0 commit comments