8388931: Shenandoah: Optimize ShenandoahHeapRegion footprint

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -355,8 +355,10 @@ jint ShenandoahHeap::initialize() {
   //
   // Create regions and region sets
   //
-  size_t region_align = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE);
-  size_t region_storage_size_orig = region_align * _num_regions;
+  size_t region_granule = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE);
+  assert(region_granule <= SHENANDOAH_CACHE_LINE_SIZE*2,
+         "Performance: Should take only a few cache lines: %zu", sizeof(ShenandoahHeapRegion));
+  size_t region_storage_size_orig = region_granule * _num_regions;
   size_t region_storage_size = align_up(region_storage_size_orig,
                                         MAX2(region_page_size, os::vm_allocation_granularity()));
 
@@ -416,7 +418,7 @@ jint ShenandoahHeap::initialize() {
     for (size_t i = 0; i < _num_regions; i++) {
       HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i;
       bool is_committed = i < num_committed_regions;
-      void* loc = region_storage.base() + i * region_align;
+      void* loc = region_storage.base() + i * region_granule;
 
       ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed);
       assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity");

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp

@@ -57,36 +57,37 @@ size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0;
 size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0;
 
 ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed) :
-  _index(index),
   _bottom(start),
   _end(start + RegionSizeWords),
+  _index(checked_cast<uint32_t>(index)),
+  _state(committed ? _empty_committed : _empty_uncommitted),
   _new_top(nullptr),
-  _empty_time(os::elapsedTime()),
   _top_before_promoted(nullptr),
   _top_at_evac_start(start),
-  _state(committed ? _empty_committed : _empty_uncommitted),
+  _coalesce_and_fill_boundary(start),
+  _mixed_candidate_garbage_words(0),
+  _live_data(0),
+  _critical_pins(0),
+  _update_watermark(start),
   _top(start),
   _tlab_allocs(0),
   _gclab_allocs(0),
   _plab_allocs(0),
-  _live_data(0),
-  _critical_pins(0),
-  _mixed_candidate_garbage_words(0),
-  _update_watermark(start),
+  _empty_time(os::elapsedTime()),
   _age(0),
-#ifdef SHENANDOAH_CENSUS_NOISE
+  #ifdef SHENANDOAH_CENSUS_NOISE
   _youth(0),
-#endif // SHENANDOAH_CENSUS_NOISE
-  _needs_bitmap_reset(false)
-  {
-
+  #endif
+  _needs_bitmap_reset(false),
+  _promoted_in_place(false)
+{
   assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end),
          "invalid space boundaries");
   if (ZapUnusedHeapArea && committed) {
     SpaceMangler::mangle_region(MemRegion(_bottom, _end));
   }
-  _recycling.unset();
-  _has_self_forwards.unset();
+  _recycling.release_store(false);
+  _has_self_forwards.release_store(false);
 }
 
 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
@@ -386,7 +387,7 @@ void ShenandoahHeapRegion::set_live_data(size_t s) {
 
 void ShenandoahHeapRegion::print_on(outputStream* st) const {
   st->print("|");
-  st->print("%5zu", this->_index);
+  st->print(UINT32_FORMAT_W(5), this->_index);
 
   switch (state()) {
     case _empty_uncommitted:
@@ -569,7 +570,7 @@ ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
 
 
 void ShenandoahHeapRegion::recycle_internal() {
-  assert(_recycling.is_set() && is_trash(), "Wrong state");
+  assert(_recycling.load_acquire() && is_trash(), "Wrong state");
   ShenandoahHeap* heap = ShenandoahHeap::heap();
 
   _top_at_evac_start = _bottom;
@@ -600,20 +601,20 @@ void ShenandoahHeapRegion::try_recycle_under_lock() {
   if (!is_trash()) {
     return;
   }
-  if (_recycling.try_set()) {
+  if (_recycling.compare_set(false, true)) {
     if (is_trash()) {
       // At freeset rebuild time, which precedes recycling of collection set, we treat all cset regions as
       // part of capacity, as empty, as fully available, and as unaffiliated.  This provides short-lived optimism
       // for triggering heuristics.  It greatly simplifies and reduces the locking overhead required
       // by more time-precise accounting of these details.
       recycle_internal();
     }
-    _recycling.unset();
+    _recycling.release_store(false);
   } else {
     // Ensure recycling is unset before returning to mutator to continue memory allocation.
     // Otherwise, the mutator might see region as fully recycled and might change its affiliation only to have
     // the racing GC worker thread overwrite its affiliation to FREE.
-    while (_recycling.is_set()) {
+    while (_recycling.load_acquire()) {
       if (os::is_MP()) {
         SpinPause();
       } else {
@@ -631,7 +632,7 @@ void ShenandoahHeapRegion::try_recycle() {
   if (!is_trash()) {
     return;
   }
-  if (_recycling.try_set()) {
+  if (_recycling.compare_set(false, true)) {
     // Double check region state after win the race to set recycling flag
     if (is_trash()) {
       // At freeset rebuild time, which precedes recycling of collection set, we treat all cset regions as
@@ -640,7 +641,7 @@ void ShenandoahHeapRegion::try_recycle() {
       // by more time-precise accounting of these details.
       recycle_internal();
     }
-    _recycling.unset();
+    _recycling.release_store(false);
   }
 }
 

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp

@@ -239,50 +239,66 @@ class ShenandoahHeapRegion {
   static size_t MaxTLABSizeBytes;
   static size_t MaxTLABSizeWords;
 
-  // Never updated fields
-  size_t const _index;
+  // NOTE: This class is allocated specially by ShenandoahHeap, starting on cache line
+  // boundary. We have to carefully manage the order of these fields and the total density
+  // of the instance. Sometimes we arrange the fields in "unnatural" order to get more
+  // compact memory representation. If you change/rearrange the fields here, check the new
+  // layout with pahole.
+
+  // We split fields in two groups to hit two different cache lines. The first cache line
+  // contains never/seldom updated fields, and also the fields we consult very often, like
+  // region state. The second cache line contains heavily updated fields, where contention
+  // is unfortunate but at least contained.
+
   HeapWord* const _bottom;
   HeapWord* const _end;
+  uint32_t const _index;
 
-  // Rarely updated fields
+  Atomic<RegionState> _state;
   HeapWord* _new_top;
-  double _empty_time;
-
   HeapWord* _top_before_promoted;
   HeapWord* _top_at_evac_start;
 
-  // Seldom updated fields
-  Atomic<RegionState> _state;
-  HeapWord* _coalesce_and_fill_boundary; // for old regions not selected as collection set candidates.
+  // For old regions not selected as collection set candidates
+  HeapWord* _coalesce_and_fill_boundary;
 
-  // Frequently updated fields
-  HeapWord* _top;
+  size_t _mixed_candidate_garbage_words;
 
-  size_t _tlab_allocs;
-  size_t _gclab_allocs;
-  size_t _plab_allocs;
+  // --- cache line boundary here --
 
+  // Live data can include humongous region size.
   Atomic<size_t> _live_data;
-  Atomic<size_t> _critical_pins;
 
-  size_t _mixed_candidate_garbage_words;
+  Atomic<size_t> _critical_pins;
 
   Atomic<HeapWord*> _update_watermark;
 
-  uint _age;
-  bool _promoted_in_place;
-  CENSUS_NOISE(uint _youth;)   // tracks epochs of retrograde ageing (rejuvenation)
+  HeapWord* _top;
+
+  // These are only for regular allocs, so they cannot be larger than a single region.
+  uint32_t _tlab_allocs;
+  uint32_t _gclab_allocs;
+  uint32_t _plab_allocs;
 
-  ShenandoahSharedFlag _recycling; // Used to indicate that the region is being recycled; see try_recycle*().
+  float _empty_time;
 
   // Set when an evacuation failure self-forwarded at least one object in this
   // region. The drain at degen/full GC entry scans flagged regions and CAS-
   // clears the self_fwd bits. Safety-net reset on region recycle.
-  ShenandoahSharedFlag _has_self_forwards;
+  Atomic<bool> _has_self_forwards;
+
+  // Used to indicate that the region is being recycled; see try_recycle*()
+  Atomic<bool> _recycling;
+
+  // Aging and census data
+  uint8_t _age;
+  CENSUS_NOISE(uint8_t _youth;)
 
   // This is only read/written by a gc worker to avoid unnecessary bitmap resets
   bool _needs_bitmap_reset;
 
+  bool _promoted_in_place;
+
 public:
   ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed);
 
@@ -535,9 +551,9 @@ class ShenandoahHeapRegion {
   // Self-forward accounting: set by an evacuating thread after it successfully
   // installs a self-forward mark on an object in this region. Tested and cleared
   // at the drain phase (degen/full GC entry) and again on region recycle.
-  bool has_self_forwards() const { return _has_self_forwards.is_set(); }
-  void set_has_self_forwards()   { _has_self_forwards.set(); }
-  void clear_has_self_forwards() { _has_self_forwards.unset(); }
+  bool has_self_forwards() const { return _has_self_forwards.load_acquire(); }
+  void set_has_self_forwards()   { _has_self_forwards.release_store(true); }
+  void clear_has_self_forwards() { _has_self_forwards.release_store(false); }
 
 private:
   void decrement_humongous_waste();

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp

@@ -75,11 +75,11 @@ inline void ShenandoahHeapRegion::adjust_alloc_metadata(const ShenandoahAllocReq
   // Only need to update alloc metadata for lab alloc, shared alloc is counted implicitly by tlab/gclab allocs
   if (req.is_lab_alloc()) {
     if (req.is_mutator_alloc()) {
-      _tlab_allocs += size;
+      _tlab_allocs = checked_cast<uint32_t>(_tlab_allocs + size);
     } else if (req.is_old()) {
-      _plab_allocs += size;
+      _plab_allocs = checked_cast<uint32_t>(_plab_allocs + size);
     } else {
-      _gclab_allocs += size;
+      _gclab_allocs = checked_cast<uint32_t>(_gclab_allocs + size);
     }
   }
 }

src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp

@@ -40,7 +40,7 @@
   static_field(ShenandoahHeapRegion, RegionSizeBytes,              size_t)                                    \
   static_field(ShenandoahHeapRegion, RegionSizeBytesShift,         size_t)                                    \
   nonstatic_field(ShenandoahHeapRegion, _state,                    Atomic<ShenandoahHeapRegion::RegionState>) \
-  nonstatic_field(ShenandoahHeapRegion, _index,                    size_t const)                              \
+  nonstatic_field(ShenandoahHeapRegion, _index,                    uint32_t const)                            \
   nonstatic_field(ShenandoahHeapRegion, _bottom,                   HeapWord* const)                           \
   nonstatic_field(ShenandoahHeapRegion, _top,                      HeapWord*)                                 \
   nonstatic_field(ShenandoahHeapRegion, _end,                      HeapWord* const)                           \