foreigh free

Author: wenyuzhao

bench/Cargo.toml

@@ -122,4 +122,4 @@ mi = { env = { MALLOC = "mi", IS_MALLOCKIT = "0" } }
 # je = { env = { MALLOC = "je", IS_MALLOCKIT = "0" } }
 
 [package.metadata.harness.profiles.default.probes]
-harness-probe-perf = { events = "PERF_COUNT_SW_TASK_CLOCK,PERF_COUNT_HW_CPU_CYCLES,PERF_COUNT_HW_INSTRUCTIONS", inherit = true }
+harness-probe-perf = { events = "PERF_COUNT_SW_TASK_CLOCK,PERF_COUNT_HW_CPU_CYCLES,PERF_COUNT_HW_INSTRUCTIONS,PERF_COUNT_SW_PAGE_FAULTS,PERF_COUNT_SW_CONTEXT_SWITCHES", inherit = true }

ix/src/block.rs

@@ -78,15 +78,12 @@ impl Block {
     pub const LINES: usize = Self::BYTES / Line::BYTES;
     #[allow(unused)]
     pub const DATA_LINES: usize = Self::DATA_BYTES / Line::BYTES;
+    pub const REUSABLE_THRESHOLD: usize = Self::LINES / 2;
 
     pub fn init(&mut self, owner: usize) {
-        // debug_assert_eq!(Self::META_BYTES, Address::BYTES * 8);
         self.owner = owner;
         self.live_lines = 0;
-        for i in 0..LINES_IN_BLOCK {
-            self.line_liveness[i] = 0;
-        }
-        self.foreign_free.store(Address::ZERO, Ordering::SeqCst);
+        self.foreign_free.store(Address::ZERO, Ordering::Relaxed);
         self.state = BlockState::Allocating;
         self.next = None;
         self.prev = None;
@@ -95,7 +92,7 @@ impl Block {
     pub fn deinit(&mut self) {
         self.state = BlockState::Free;
         self.live_lines = 0;
-        self.foreign_free.store(Address::ZERO, Ordering::SeqCst);
+        self.foreign_free.store(Address::ZERO, Ordering::Relaxed);
         self.next = None;
         self.prev = None;
     }
@@ -132,7 +129,8 @@ impl Block {
             }
             cursor += 1;
         }
-        if cursor != start_cursor {
+        let first_cursor = self.lines().start.get_index_within_block();
+        if cursor != start_cursor && cursor != first_cursor {
             cursor += 1;
         }
         if cursor >= self.line_liveness.len() {
@@ -191,6 +189,7 @@ impl Block {
             self.line_liveness[i] += 1;
         }
         self.live_lines += lines;
+        // self.drain_foreign_free();
         // println!(" - BA {:x?} live-lines {}", self, self.live_lines);
     }
 
@@ -237,14 +236,42 @@ impl Block {
             // println!(" - S-{:?} dead-{}", self.state, dead_lines);
             if dead_lines >= 1 && self.state == BlockState::Full {
                 let live_lines = self.live_lines;
-                if live_lines < Block::DATA_LINES / 2 {
+                if live_lines <= Block::REUSABLE_THRESHOLD {
                     let owner = &mut crate::ImmixMutator::current().ix;
                     owner.add_reusable_block(*self);
                 }
             }
         }
     }
 
+    pub fn drain_foreign_free(&mut self) {
+        let mut ptr;
+        loop {
+            ptr = self.foreign_free.load(Ordering::Relaxed);
+            if ptr.is_zero() {
+                break;
+            }
+            if self
+                .foreign_free
+                .compare_exchange(ptr, Address::ZERO, Ordering::SeqCst, Ordering::SeqCst)
+                .is_ok()
+            {
+                break;
+            }
+        }
+        // let mut count = 0;
+        while !ptr.is_zero() {
+            let next = unsafe { ptr.load() };
+            self.dealloc_impl(ptr, self.get_layout(ptr));
+            ptr = next;
+            // count += 1;
+        }
+        // if count > 0 {
+        //     //
+        //     println!(" - DF {:x?} freed {}", self, count);
+        // }
+    }
+
     #[inline]
     pub fn on_dealloc(&mut self, ptr: Address, layout: Layout) {
         // println!("FLocal {:?}", ptr..(ptr + layout.size()));

ix/src/immix_space.rs

@@ -110,6 +110,7 @@ impl ImmixAllocator {
         if let Some(mut next) = next {
             next.prev = None;
         }
+        b.drain_foreign_free();
         self.reusable_blocks = next;
         // println!(
         //     " - acquire_reusable_block {:x?} next = {:x?}",
@@ -134,7 +135,7 @@ impl ImmixAllocator {
                     debug_assert!(b.prev.is_none());
                     debug_assert_ne!(Some(b), self.reusable_blocks);
                     self.space.release_block(b);
-                } else if live_lines < Block::DATA_LINES / 2 {
+                } else if live_lines <= Block::REUSABLE_THRESHOLD {
                     self.add_reusable_block(b);
                 } else {
                     b.state = BlockState::Full;
@@ -257,6 +258,8 @@ impl Allocator for ImmixAllocator {
         let result = self.cursor;
         let new_cursor = self.cursor + layout.size();
         let result = if new_cursor > self.limit {
+            self.large_block.map(|mut b| b.drain_foreign_free());
+            self.block.map(|mut b| b.drain_foreign_free());
             if layout.size() > Line::BYTES {
                 // Size larger than a line: do large allocation
                 self.overflow_alloc(layout)
@@ -289,5 +292,11 @@ impl Drop for ImmixAllocator {
     fn drop(&mut self) {
         // self.tlab
         //     .clear(|cell| self.local.free_cell(cell, self.space));
+        // println!(" - ImmixAllocator.drop {:x?}", self as *const Self);
+        let mut reusable_blocks = self.reusable_blocks;
+        while let Some(mut block) = reusable_blocks {
+            reusable_blocks = block.next;
+            block.drain_foreign_free();
+        }
     }
 }