@@ -172,6 +172,9 @@ impl<T> Segment<T> {
172172 // SAFETY: This is an intermediate segment, so we can safely drop the children segments.
173173 let guard = unsafe { crossbeam_epoch:: unprotected ( ) } ;
174174 for child in unsafe { & self . children } . iter ( ) {
175+ if child. load ( Relaxed , guard) . is_null ( ) {
176+ continue ; // skip null children
177+ }
175178 unsafe {
176179 let child_seg = child. load ( Relaxed , guard) . into_owned ( ) ;
177180 child_seg. into_box ( ) . deallocate ( height - 1 ) ;
@@ -208,13 +211,22 @@ impl<T> Default for GrowableArray<T> {
208211 }
209212}
210213
214+ /// Convert an index to a vector of segments.
215+ ///
216+ /// For example, if `SEGMENT_LOGSIZE = 3`, then the index `0b111011` will be converted to
217+ /// `vec![0b111, 0b011]`, which corresponds to the segments at height 2 and 1 respectively.
218+ #[ inline]
211219fn get_idx_seg_vec ( index : usize ) -> Vec < usize > {
212220 let mut index_seg_vec = Vec :: new ( ) ;
213221 let mut index = index;
214- while index > 0 {
222+ loop {
215223 index_seg_vec. push ( index & ( ( 1 << SEGMENT_LOGSIZE ) - 1 ) ) ;
216224 index >>= SEGMENT_LOGSIZE ;
225+ if index == 0 {
226+ break ;
227+ }
217228 }
229+ index_seg_vec. reverse ( ) ;
218230 index_seg_vec
219231}
220232
@@ -226,60 +238,110 @@ impl<T> GrowableArray<T> {
226238 }
227239 }
228240
229- /// Returns the reference to the `Atomic` pointer at `index`. Allocates new segments if
230- /// necessary.
231- pub fn get < ' g > ( & self , index : usize , guard : & ' g Guard ) -> & ' g Atomic < T > {
232- let mask = ( 1 << SEGMENT_LOGSIZE ) - 1 ;
233- let index_seg_vec = get_idx_seg_vec ( index) ;
234- let h_required = index_seg_vec. len ( ) ;
235-
241+ /// Increase the height of the root segment to at least `h_required`.
242+ fn increase_height_to_needed ( & self , h_required : usize , guard : & Guard ) {
236243 let mut root_seg = self . root . load ( SeqCst , guard) ;
237244 while root_seg. tag ( ) < h_required {
238245 // Allocate a new segment and set it as the root.
239246 let new_seg = Segment :: < T > :: new ( ) . with_tag ( root_seg. tag ( ) + 1 ) ;
240- if self
247+ match self
241248 . root
242249 . compare_exchange ( root_seg, new_seg, SeqCst , Relaxed , guard)
243- . is_ok ( )
244250 {
245- // updated root
246- root_seg = self . root . load ( SeqCst , guard) ;
247- } else {
248- root_seg = self . root . load ( SeqCst , guard) ;
251+ Ok ( mut new) => {
252+ if root_seg. tag ( ) == 0 {
253+ root_seg = new;
254+ unsafe {
255+ root_seg. deref_mut ( ) . children [ 0 ] . store ( Shared :: null ( ) , SeqCst ) ;
256+ }
257+ continue ;
258+ }
259+ unsafe {
260+ new. deref_mut ( ) . children [ 0 ] . store ( root_seg, SeqCst ) ;
261+ }
262+ // updated root
263+ root_seg = new;
264+ }
265+ Err ( e) => {
266+ // Another thread has already set this root, so we load it.
267+ root_seg = e. current ;
268+ }
249269 }
250270 }
271+ }
272+
273+ /// If the height of the root segment is larger than `h_required`, then return the segment whose
274+ /// height is equal to `h_required`.
275+ fn find_suitable_root < ' g > (
276+ & self ,
277+ h_required : usize ,
278+ guard : & ' g Guard ,
279+ ) -> Shared < ' g , Segment < T > > {
280+ let mut root_seg = self . root . load ( SeqCst , guard) ;
281+ while root_seg. tag ( ) > h_required {
282+ unsafe {
283+ let children = & root_seg. as_ref ( ) . unwrap ( ) . children ;
284+ // Get the first child segment, which is guaranteed to exist since we just increased
285+ // the height of the root segment.
286+ root_seg = children[ 0 ] . load ( SeqCst , guard) ;
287+ }
288+ }
289+ if root_seg. tag ( ) < h_required {
290+ panic ! (
291+ "GrowableArray::find_suitable_root: height {} is larger than the root segment's height {}" ,
292+ h_required,
293+ root_seg. tag( )
294+ ) ;
295+ }
296+ root_seg
297+ }
298+
299+ /// Returns the reference to the `Atomic` pointer at `index`. Allocates new segments if
300+ /// necessary.
301+ pub fn get < ' g > ( & self , index : usize , guard : & ' g Guard ) -> & ' g Atomic < T > {
302+ let index_seg_vec = get_idx_seg_vec ( index) ;
303+ let h_required = index_seg_vec. len ( ) ;
304+
305+ self . increase_height_to_needed ( h_required, guard) ;
306+ let mut seg = self . find_suitable_root ( h_required, guard) ;
251307
252- let mut seg = root_seg;
253- for ( i, index_seg) in index_seg_vec. iter ( ) . rev ( ) . enumerate ( ) {
254- if i == h_required - 1 {
308+ for index_seg in index_seg_vec {
309+ if seg. tag ( ) == 1 {
255310 // This is the last segment, so we return the element.
256311 let elements = unsafe { & seg. as_ref ( ) . unwrap ( ) . elements } ;
257- let element = & elements[ * index_seg] ;
258- return element;
312+ return & elements[ index_seg] ;
259313 }
260314 // This is an intermediate segment, so we traverse to the next segment.
261315 let children = unsafe { & seg. as_ref ( ) . unwrap ( ) . children } ;
262- let child_seg = children[ * index_seg] . load ( SeqCst , guard) ;
316+ let child_seg = children[ index_seg] . load ( SeqCst , guard) ;
263317 if child_seg. is_null ( ) {
264318 // Allocate a new segment and set it as the child.
265319 let new_child_seg = Segment :: < T > :: new ( ) . with_tag ( seg. tag ( ) - 1 ) ;
266- if children[ * index_seg]
267- . compare_exchange ( child_seg, new_child_seg, SeqCst , Relaxed , guard)
268- . is_ok ( )
269- {
270- seg = children[ * index_seg] . load ( SeqCst , guard) ;
271- } else {
272- // updated child
273- seg = children[ * index_seg] . load ( SeqCst , guard) ;
320+ match children[ index_seg] . compare_exchange (
321+ child_seg,
322+ new_child_seg,
323+ SeqCst ,
324+ Relaxed ,
325+ guard,
326+ ) {
327+ Ok ( new) => {
328+ seg = new; // updated child
329+ }
330+ Err ( e) => {
331+ // Another thread has already set this child, so we load it.
332+ seg = e. current ;
333+ }
274334 }
275335 } else {
276336 seg = child_seg;
277337 }
278338 }
279339
280340 panic ! (
281- "GrowableArray::get: index {} is out of bounds for height {}" ,
282- index, h_required
341+ "GrowableArray::get: index (0x{:X}) is out of bounds for height {} at seg height {}" ,
342+ index,
343+ h_required,
344+ seg. tag( )
283345 ) ;
284346 }
285347}
0 commit comments