diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 84add03f1bac0..d5677c5a14ac8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -355,8 +355,10 @@ jint ShenandoahHeap::initialize() { // // Create regions and region sets // - size_t region_align = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE); - size_t region_storage_size_orig = region_align * _num_regions; + size_t region_granule = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE); + assert(region_granule <= SHENANDOAH_CACHE_LINE_SIZE*2, + "Performance: Should take only a few cache lines: %zu", sizeof(ShenandoahHeapRegion)); + size_t region_storage_size_orig = region_granule * _num_regions; size_t region_storage_size = align_up(region_storage_size_orig, MAX2(region_page_size, os::vm_allocation_granularity())); @@ -416,7 +418,7 @@ jint ShenandoahHeap::initialize() { for (size_t i = 0; i < _num_regions; i++) { HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i; bool is_committed = i < num_committed_regions; - void* loc = region_storage.base() + i * region_align; + void* loc = region_storage.base() + i * region_granule; ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed); assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 66eaed2c22218..488b48d9ad0f1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -57,36 +57,37 @@ size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0; size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0; ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed) : - _index(index), _bottom(start), _end(start + RegionSizeWords), + _index(checked_cast(index)), + _state(committed ? _empty_committed : _empty_uncommitted), _new_top(nullptr), - _empty_time(os::elapsedTime()), _top_before_promoted(nullptr), _top_at_evac_start(start), - _state(committed ? _empty_committed : _empty_uncommitted), + _coalesce_and_fill_boundary(start), + _mixed_candidate_garbage_words(0), + _live_data(0), + _critical_pins(0), + _update_watermark(start), _top(start), _tlab_allocs(0), _gclab_allocs(0), _plab_allocs(0), - _live_data(0), - _critical_pins(0), - _mixed_candidate_garbage_words(0), - _update_watermark(start), + _empty_time(os::elapsedTime()), _age(0), -#ifdef SHENANDOAH_CENSUS_NOISE + #ifdef SHENANDOAH_CENSUS_NOISE _youth(0), -#endif // SHENANDOAH_CENSUS_NOISE - _needs_bitmap_reset(false) - { - + #endif + _needs_bitmap_reset(false), + _promoted_in_place(false) +{ assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end), "invalid space boundaries"); if (ZapUnusedHeapArea && committed) { SpaceMangler::mangle_region(MemRegion(_bottom, _end)); } - _recycling.unset(); - _has_self_forwards.unset(); + _recycling.release_store(false); + _has_self_forwards.release_store(false); } void ShenandoahHeapRegion::report_illegal_transition(const char *method) { @@ -386,7 +387,7 @@ void ShenandoahHeapRegion::set_live_data(size_t s) { void ShenandoahHeapRegion::print_on(outputStream* st) const { st->print("|"); - st->print("%5zu", this->_index); + st->print(UINT32_FORMAT_W(5), this->_index); switch (state()) { case _empty_uncommitted: @@ -569,7 +570,7 @@ ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const { void ShenandoahHeapRegion::recycle_internal() { - assert(_recycling.is_set() && is_trash(), "Wrong state"); + assert(_recycling.load_acquire() && is_trash(), "Wrong state"); ShenandoahHeap* heap = ShenandoahHeap::heap(); _top_at_evac_start = _bottom; @@ -600,7 +601,7 @@ void ShenandoahHeapRegion::try_recycle_under_lock() { if (!is_trash()) { return; } - if (_recycling.try_set()) { + if (_recycling.compare_set(false, true)) { if (is_trash()) { // At freeset rebuild time, which precedes recycling of collection set, we treat all cset regions as // part of capacity, as empty, as fully available, and as unaffiliated. This provides short-lived optimism @@ -608,12 +609,12 @@ void ShenandoahHeapRegion::try_recycle_under_lock() { // by more time-precise accounting of these details. recycle_internal(); } - _recycling.unset(); + _recycling.release_store(false); } else { // Ensure recycling is unset before returning to mutator to continue memory allocation. // Otherwise, the mutator might see region as fully recycled and might change its affiliation only to have // the racing GC worker thread overwrite its affiliation to FREE. - while (_recycling.is_set()) { + while (_recycling.load_acquire()) { if (os::is_MP()) { SpinPause(); } else { @@ -631,7 +632,7 @@ void ShenandoahHeapRegion::try_recycle() { if (!is_trash()) { return; } - if (_recycling.try_set()) { + if (_recycling.compare_set(false, true)) { // Double check region state after win the race to set recycling flag if (is_trash()) { // At freeset rebuild time, which precedes recycling of collection set, we treat all cset regions as @@ -640,7 +641,7 @@ void ShenandoahHeapRegion::try_recycle() { // by more time-precise accounting of these details. recycle_internal(); } - _recycling.unset(); + _recycling.release_store(false); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index cf60f18e48f3e..fb2b40b6cb6eb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -239,50 +239,66 @@ class ShenandoahHeapRegion { static size_t MaxTLABSizeBytes; static size_t MaxTLABSizeWords; - // Never updated fields - size_t const _index; + // NOTE: This class is allocated specially by ShenandoahHeap, starting on cache line + // boundary. We have to carefully manage the order of these fields and the total density + // of the instance. Sometimes we arrange the fields in "unnatural" order to get more + // compact memory representation. If you change/rearrange the fields here, check the new + // layout with pahole. + + // We split fields in two groups to hit two different cache lines. The first cache line + // contains never/seldom updated fields, and also the fields we consult very often, like + // region state. The second cache line contains heavily updated fields, where contention + // is unfortunate but at least contained. + HeapWord* const _bottom; HeapWord* const _end; + uint32_t const _index; - // Rarely updated fields + Atomic _state; HeapWord* _new_top; - double _empty_time; - HeapWord* _top_before_promoted; HeapWord* _top_at_evac_start; - // Seldom updated fields - Atomic _state; - HeapWord* _coalesce_and_fill_boundary; // for old regions not selected as collection set candidates. + // For old regions not selected as collection set candidates + HeapWord* _coalesce_and_fill_boundary; - // Frequently updated fields - HeapWord* _top; + size_t _mixed_candidate_garbage_words; - size_t _tlab_allocs; - size_t _gclab_allocs; - size_t _plab_allocs; + // --- cache line boundary here -- + // Live data can include humongous region size. Atomic _live_data; - Atomic _critical_pins; - size_t _mixed_candidate_garbage_words; + Atomic _critical_pins; Atomic _update_watermark; - uint _age; - bool _promoted_in_place; - CENSUS_NOISE(uint _youth;) // tracks epochs of retrograde ageing (rejuvenation) + HeapWord* _top; + + // These are only for regular allocs, so they cannot be larger than a single region. + uint32_t _tlab_allocs; + uint32_t _gclab_allocs; + uint32_t _plab_allocs; - ShenandoahSharedFlag _recycling; // Used to indicate that the region is being recycled; see try_recycle*(). + float _empty_time; // Set when an evacuation failure self-forwarded at least one object in this // region. The drain at degen/full GC entry scans flagged regions and CAS- // clears the self_fwd bits. Safety-net reset on region recycle. - ShenandoahSharedFlag _has_self_forwards; + Atomic _has_self_forwards; + + // Used to indicate that the region is being recycled; see try_recycle*() + Atomic _recycling; + + // Aging and census data + uint8_t _age; + CENSUS_NOISE(uint8_t _youth;) // This is only read/written by a gc worker to avoid unnecessary bitmap resets bool _needs_bitmap_reset; + bool _promoted_in_place; + public: ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed); @@ -535,9 +551,9 @@ class ShenandoahHeapRegion { // Self-forward accounting: set by an evacuating thread after it successfully // installs a self-forward mark on an object in this region. Tested and cleared // at the drain phase (degen/full GC entry) and again on region recycle. - bool has_self_forwards() const { return _has_self_forwards.is_set(); } - void set_has_self_forwards() { _has_self_forwards.set(); } - void clear_has_self_forwards() { _has_self_forwards.unset(); } + bool has_self_forwards() const { return _has_self_forwards.load_acquire(); } + void set_has_self_forwards() { _has_self_forwards.release_store(true); } + void clear_has_self_forwards() { _has_self_forwards.release_store(false); } private: void decrement_humongous_waste(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index f004fdf0ea2be..933b7e286fae3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -75,11 +75,11 @@ inline void ShenandoahHeapRegion::adjust_alloc_metadata(const ShenandoahAllocReq // Only need to update alloc metadata for lab alloc, shared alloc is counted implicitly by tlab/gclab allocs if (req.is_lab_alloc()) { if (req.is_mutator_alloc()) { - _tlab_allocs += size; + _tlab_allocs = checked_cast(_tlab_allocs + size); } else if (req.is_old()) { - _plab_allocs += size; + _plab_allocs = checked_cast(_plab_allocs + size); } else { - _gclab_allocs += size; + _gclab_allocs = checked_cast(_gclab_allocs + size); } } } diff --git a/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp b/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp index 4299bdb8126fc..cab1abad10ffa 100644 --- a/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp +++ b/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp @@ -40,7 +40,7 @@ static_field(ShenandoahHeapRegion, RegionSizeBytes, size_t) \ static_field(ShenandoahHeapRegion, RegionSizeBytesShift, size_t) \ nonstatic_field(ShenandoahHeapRegion, _state, Atomic) \ - nonstatic_field(ShenandoahHeapRegion, _index, size_t const) \ + nonstatic_field(ShenandoahHeapRegion, _index, uint32_t const) \ nonstatic_field(ShenandoahHeapRegion, _bottom, HeapWord* const) \ nonstatic_field(ShenandoahHeapRegion, _top, HeapWord*) \ nonstatic_field(ShenandoahHeapRegion, _end, HeapWord* const) \