@@ -10,13 +10,18 @@ use core::{
10
10
ops:: { Deref , DerefMut , Index , IndexMut } ,
11
11
ptr,
12
12
slice:: SliceIndex ,
13
- sync:: atomic:: { AtomicUsize , Ordering } ,
14
13
} ;
15
14
15
+ #[ cfg( feature = "atomic_append" ) ]
16
+ use core:: sync:: atomic:: { AtomicUsize , Ordering } ;
17
+
16
18
struct HeaderVecHeader < H > {
17
19
head : H ,
18
20
capacity : usize ,
21
+ #[ cfg( feature = "atomic_append" ) ]
19
22
len : AtomicUsize ,
23
+ #[ cfg( not( feature = "atomic_append" ) ) ]
24
+ len : usize ,
20
25
}
21
26
22
27
/// A vector with a header of your choosing behind a thin pointer
@@ -75,82 +80,94 @@ impl<H, T> HeaderVec<H, T> {
75
80
unsafe { core:: ptr:: write ( & mut header. head , head) } ;
76
81
// These primitive types don't have drop implementations.
77
82
header. capacity = capacity;
78
- header. len = AtomicUsize :: new ( 0 ) ;
83
+ header. len = 0usize . into ( ) ;
79
84
80
85
this
81
86
}
82
87
83
- /// Get the length of the vector from a mutable reference.
88
+ /// Get the length of the vector from a mutable reference. When one has a `&mut
89
+ /// HeaderVec`, this is the method is always exact and can be slightly faster than the non
90
+ /// mutable `len()`.
91
+ #[ cfg( feature = "atomic_append" ) ]
84
92
#[ inline( always) ]
85
- pub fn len_from_mut ( & mut self ) -> usize {
93
+ pub fn len_exact ( & mut self ) -> usize {
86
94
* self . header_mut ( ) . len . get_mut ( )
87
95
}
88
-
89
- /// Get the length of the vector with `Ordering::Acquire`. This ensures that the length is
90
- /// properly synchronized after it got atomically updated.
96
+ #[ cfg( not( feature = "atomic_append" ) ) ]
91
97
#[ inline( always) ]
92
- pub fn len_atomic_acquire ( & self ) -> usize {
93
- self . header ( ) . len . load ( Ordering :: Acquire )
98
+ pub fn len_exact ( & mut self ) -> usize {
99
+ self . header_mut ( ) . len
94
100
}
95
101
96
- /// Get the length of the vector with `Ordering::Relaxed`. This is useful for when you don't
97
- /// need exact synchronization semantic.
102
+ /// This gives the length of the `HeaderVec`. This is the non synchronized variant may
103
+ /// produce racy results in case another thread atomically appended to
104
+ /// `&self`. Nevertheless it is always safe to use.
105
+ #[ cfg( feature = "atomic_append" ) ]
98
106
#[ inline( always) ]
99
- pub fn len_atomic_relaxed ( & self ) -> usize {
100
- self . header ( ) . len . load ( Ordering :: Relaxed )
107
+ pub fn len ( & self ) -> usize {
108
+ self . len_atomic_relaxed ( )
101
109
}
102
-
103
- /// Alias for [`HeaderVec::len_atomic_relaxed`]. This gives always a valid view of the
104
- /// length when used in isolation.
110
+ #[ cfg( not( feature = "atomic_append" ) ) ]
105
111
#[ inline( always) ]
106
112
pub fn len ( & self ) -> usize {
107
- self . len_atomic_relaxed ( )
113
+ self . header ( ) . len
108
114
}
109
115
110
- /// Add `n` to the length of the vector atomically with `Ordering::Release`.
111
- ///
112
- /// # Safety
113
- ///
114
- /// Before incrementing the length of the vector, you must ensure that new elements are
115
- /// properly initialized.
116
+ /// This gives the length of the `HeaderVec`. With `atomic_append` enabled this gives a
117
+ /// exact result *after* another thread atomically appended to this `HeaderVec`. It still
118
+ /// requires synchronization because the length may become invalidated when another thread
119
+ /// atomically appends data to this `HeaderVec` while we still work with the result of
120
+ /// this method.
121
+ # [ cfg ( not ( feature = "atomic_append" ) ) ]
116
122
#[ inline( always) ]
117
- pub unsafe fn len_atomic_add_release ( & self , n : usize ) -> usize {
118
- self . header ( ) . len . fetch_add ( n, Ordering :: Release )
123
+ pub fn len_strict ( & self ) -> usize {
124
+ self . header ( ) . len
125
+ }
126
+ #[ cfg( feature = "atomic_append" ) ]
127
+ #[ inline( always) ]
128
+ pub fn len_strict ( & self ) -> usize {
129
+ self . len_atomic_acquire ( )
119
130
}
120
131
132
+ /// Check whenever a `HeaderVec` is empty. This uses a `&mut self` reference and is
133
+ /// always exact and may be slightly faster than the non mutable variant.
121
134
#[ inline( always) ]
122
- pub fn is_empty_from_mut ( & mut self ) -> bool {
123
- self . len_from_mut ( ) == 0
135
+ pub fn is_empty_exact ( & mut self ) -> bool {
136
+ self . len_exact ( ) == 0
124
137
}
125
138
139
+ /// Check whenever a `HeaderVec` is empty. This uses a `&self` reference and may be racy
140
+ /// when another thread atomically appended to this `HeaderVec`.
126
141
#[ inline( always) ]
127
142
pub fn is_empty ( & self ) -> bool {
128
- self . len_atomic_relaxed ( ) == 0
143
+ self . len ( ) == 0
129
144
}
130
145
146
+ /// Check whenever a `HeaderVec` is empty. see [`len_strict()`] about the exactness guarantees.
131
147
#[ inline( always) ]
132
- pub fn is_empty_atomic_acquire ( & self ) -> bool {
133
- self . len_atomic_acquire ( ) == 0
148
+ pub fn is_empty_strict ( & self ) -> bool {
149
+ self . len_strict ( ) == 0
134
150
}
135
151
136
152
#[ inline( always) ]
137
153
pub fn capacity ( & self ) -> usize {
138
154
self . header ( ) . capacity
139
155
}
140
156
157
+ /// This is the amount of elements that can be added to the `HeaderVec` without reallocation.
141
158
#[ inline( always) ]
142
- pub fn as_slice ( & self ) -> & [ T ] {
143
- unsafe { core :: slice :: from_raw_parts ( self . start_ptr ( ) , self . len_atomic_relaxed ( ) ) }
159
+ pub fn spare_capacity ( & self ) -> usize {
160
+ self . header ( ) . capacity - self . len_strict ( )
144
161
}
145
162
146
163
#[ inline( always) ]
147
- pub fn as_slice_atomic_acquire ( & self ) -> & [ T ] {
148
- unsafe { core:: slice:: from_raw_parts ( self . start_ptr ( ) , self . len_atomic_acquire ( ) ) }
164
+ pub fn as_slice ( & self ) -> & [ T ] {
165
+ unsafe { core:: slice:: from_raw_parts ( self . start_ptr ( ) , self . len_strict ( ) ) }
149
166
}
150
167
151
168
#[ inline( always) ]
152
169
pub fn as_mut_slice ( & mut self ) -> & mut [ T ] {
153
- unsafe { core:: slice:: from_raw_parts_mut ( self . start_ptr_mut ( ) , self . len_from_mut ( ) ) }
170
+ unsafe { core:: slice:: from_raw_parts_mut ( self . start_ptr_mut ( ) , self . len_exact ( ) ) }
154
171
}
155
172
156
173
/// This is useful to check if two nodes are the same. Use it with [`HeaderVec::is`].
@@ -241,7 +258,7 @@ impl<H, T> HeaderVec<H, T> {
241
258
/// Returns `true` if the memory was moved to a new location.
242
259
/// In this case, you are responsible for updating the weak nodes.
243
260
pub fn push ( & mut self , item : T ) -> Option < * const ( ) > {
244
- let old_len = self . len_from_mut ( ) ;
261
+ let old_len = self . len_exact ( ) ;
245
262
let new_len = old_len + 1 ;
246
263
let old_capacity = self . capacity ( ) ;
247
264
// If it isn't big enough.
@@ -266,7 +283,7 @@ impl<H, T> HeaderVec<H, T> {
266
283
// This keeps track of the length (and next position) of the contiguous retained elements
267
284
// at the beginning of the vector.
268
285
let mut head = 0 ;
269
- let original_len = self . len_from_mut ( ) ;
286
+ let original_len = self . len_exact ( ) ;
270
287
// Get the offset of the beginning of the slice.
271
288
let start_ptr = self . start_ptr_mut ( ) ;
272
289
// Go through each index.
@@ -346,11 +363,58 @@ impl<H, T> HeaderVec<H, T> {
346
363
}
347
364
}
348
365
366
+ #[ cfg( feature = "atomic_append" ) ]
367
+ /// The atomic append API is only enabled when the `atomic_append` feature flag is set (which
368
+ /// is the default).
369
+ impl < H , T > HeaderVec < H , T > {
370
+ /// Get the length of the vector with `Ordering::Acquire`. This ensures that the length is
371
+ /// properly synchronized after it got atomically updated.
372
+ #[ inline( always) ]
373
+ fn len_atomic_acquire ( & self ) -> usize {
374
+ self . header ( ) . len . load ( Ordering :: Acquire )
375
+ }
376
+
377
+ /// Get the length of the vector with `Ordering::Relaxed`. This is useful for when you don't
378
+ /// need exact synchronization semantic.
379
+ #[ inline( always) ]
380
+ fn len_atomic_relaxed ( & self ) -> usize {
381
+ self . header ( ) . len . load ( Ordering :: Relaxed )
382
+ }
383
+
384
+ /// Add `n` to the length of the vector atomically with `Ordering::Release`.
385
+ ///
386
+ /// # Safety
387
+ ///
388
+ /// Before incrementing the length of the vector, you must ensure that new elements are
389
+ /// properly initialized.
390
+ #[ inline( always) ]
391
+ unsafe fn len_atomic_add_release ( & self , n : usize ) -> usize {
392
+ self . header ( ) . len . fetch_add ( n, Ordering :: Release )
393
+ }
394
+
395
+ #[ inline( always) ]
396
+ pub fn is_empty_atomic_acquire ( & self ) -> bool {
397
+ self . len_atomic_acquire ( ) == 0
398
+ }
399
+
400
+ #[ inline( always) ]
401
+ pub fn as_slice_atomic_acquire ( & self ) -> & [ T ] {
402
+ unsafe { core:: slice:: from_raw_parts ( self . start_ptr ( ) , self . len_atomic_acquire ( ) ) }
403
+ }
404
+
405
+ /// Gets the pointer to the end of the slice. This returns a mutable pointer to
406
+ /// uninitialized memory behind the last element.
407
+ #[ inline( always) ]
408
+ fn end_ptr_atomic_mut ( & self ) -> * mut T {
409
+ unsafe { self . ptr . add ( Self :: offset ( ) ) . add ( self . len_atomic_acquire ( ) ) }
410
+ }
411
+ }
412
+
349
413
impl < H , T > Drop for HeaderVec < H , T > {
350
414
fn drop ( & mut self ) {
351
415
unsafe {
352
416
ptr:: drop_in_place ( & mut self . header_mut ( ) . head ) ;
353
- for ix in 0 ..self . len_from_mut ( ) {
417
+ for ix in 0 ..self . len_exact ( ) {
354
418
ptr:: drop_in_place ( self . start_ptr_mut ( ) . add ( ix) ) ;
355
419
}
356
420
alloc:: alloc:: dealloc ( self . ptr as * mut u8 , Self :: layout ( self . capacity ( ) ) ) ;
@@ -412,8 +476,7 @@ where
412
476
T : Clone ,
413
477
{
414
478
fn clone ( & self ) -> Self {
415
- let mut new_vec =
416
- Self :: with_capacity ( self . len_atomic_acquire ( ) , self . header ( ) . head . clone ( ) ) ;
479
+ let mut new_vec = Self :: with_capacity ( self . len_strict ( ) , self . header ( ) . head . clone ( ) ) ;
417
480
for e in self . as_slice ( ) {
418
481
new_vec. push ( e. clone ( ) ) ;
419
482
}
0 commit comments